From commits-return-21821-archive-asf-public=cust-asf.ponee.io@accumulo.apache.org Tue May 8 18:15:45 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id DD6FF18067B for ; Tue, 8 May 2018 18:15:44 +0200 (CEST) Received: (qmail 88854 invoked by uid 500); 8 May 2018 16:15:44 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 88842 invoked by uid 99); 8 May 2018 16:15:44 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 May 2018 16:15:44 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 6F05881AAE; Tue, 8 May 2018 16:15:43 +0000 (UTC) Date: Tue, 08 May 2018 16:15:43 +0000 To: "commits@accumulo.apache.org" Subject: [accumulo] 01/01: Merge branch '1.9' MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: kturner@apache.org In-Reply-To: <152579614253.16593.10124394667266926679@gitbox.apache.org> References: <152579614253.16593.10124394667266926679@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: accumulo X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Rev: 49192c9fe45de9efdac88afd0c3c96eb82822721 X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20180508161543.6F05881AAE@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 49192c9fe45de9efdac88afd0c3c96eb82822721 Merge: 2d869d3 b8f574f Author: Keith Turner AuthorDate: Tue May 8 12:13:44 2018 -0400 Merge branch '1.9' .../org/apache/accumulo/tserver/TabletServer.java | 10 +- .../accumulo/tserver/log/CloseableIterator.java | 25 ++ .../org/apache/accumulo/tserver/log/DfsLogger.java | 8 +- .../{MultiReader.java => RecoveryLogReader.java} | 162 ++++++++- .../accumulo/tserver/log/RecoveryLogsIterator.java | 100 ++++++ .../accumulo/tserver/log/SortedLogRecovery.java | 378 +++++++++++---------- .../apache/accumulo/tserver/logger/LogFileKey.java | 46 +-- .../apache/accumulo/tserver/logger/LogReader.java | 16 +- .../tserver/replication/AccumuloReplicaSystem.java | 6 +- .../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +- .../apache/accumulo/tserver/log/LogEventsTest.java | 36 ++ .../accumulo/tserver/log/LogFileKeyTest.java | 98 ++++++ ...ReaderTest.java => RecoveryLogsReaderTest.java} | 43 ++- .../tserver/log/SortedLogRecoveryTest.java | 296 +++++++++++++++- .../accumulo/tserver/logger/LogFileTest.java | 14 +- .../replication/AccumuloReplicaSystemTest.java | 33 +- .../BatchWriterReplicationReplayerTest.java | 4 +- .../UnusedWalDoesntCloseReplicationStatusIT.java | 2 +- 18 files changed, 991 insertions(+), 288 deletions(-) diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java index 914544a,bdf5210..708f843 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java @@@ -35,7 -35,6 +35,8 @@@ import java.util.Objects import java.util.Set; import java.util.TreeMap; +import org.apache.accumulo.core.client.impl.Table; ++import org.apache.accumulo.core.client.impl.Table.ID; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.data.impl.KeyExtent; @@@ -675,6 -679,85 +681,85 @@@ public class SortedLogRecoveryTest Assert.assertEquals(m, mutations.get(0)); } + @Test + public void testLeaveAndComeBack() throws IOException { + /** + * This test recreates the situation in bug #449 (Github issues). + */ + Mutation m1 = new ServerMutation(new Text("r1")); + m1.put("f1", "q1", "v1"); + + Mutation m2 = new ServerMutation(new Text("r2")); + m2.put("f1", "q1", "v2"); + + KeyValue entries1[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"), + createKeyValue(DEFINE_TABLET, 100, 10, extent), createKeyValue(MUTATION, 100, 10, m1), + createKeyValue(COMPACTION_START, 101, 10, "/t/f1"), + createKeyValue(COMPACTION_FINISH, 102, 10, null)}; + + KeyValue entries2[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"), + createKeyValue(DEFINE_TABLET, 1, 20, extent), createKeyValue(MUTATION, 1, 20, m2)}; + + Arrays.sort(entries1); + Arrays.sort(entries2); + Map logs = new TreeMap<>(); + logs.put("entries1", entries1); + logs.put("entries2", entries2); + + List mutations = recover(logs, extent); + + Assert.assertEquals(1, mutations.size()); + Assert.assertEquals(m2, mutations.get(0)); + } + + @Test + public void testMultipleTablets() throws IOException { - KeyExtent e1 = new KeyExtent("1", new Text("m"), null); - KeyExtent e2 = new KeyExtent("1", null, new Text("m")); ++ KeyExtent e1 = new KeyExtent(ID.of("1"), new Text("m"), null); ++ KeyExtent e2 = new KeyExtent(ID.of("1"), null, new Text("m")); + + Mutation m1 = new ServerMutation(new Text("b")); + m1.put("f1", "q1", "v1"); + + Mutation m2 = new ServerMutation(new Text("b")); + m2.put("f1", "q2", "v2"); + + Mutation m3 = new ServerMutation(new Text("s")); + m3.put("f1", "q1", "v3"); + + Mutation m4 = new ServerMutation(new Text("s")); + m4.put("f1", "q2", "v4"); + + KeyValue entries1[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"), + createKeyValue(DEFINE_TABLET, 7, 10, e1), createKeyValue(DEFINE_TABLET, 5, 11, e2), + createKeyValue(MUTATION, 8, 10, m1), createKeyValue(COMPACTION_START, 9, 10, "/t/f1"), + createKeyValue(MUTATION, 10, 10, m2), createKeyValue(COMPACTION_FINISH, 11, 10, null), + createKeyValue(MUTATION, 6, 11, m3), createKeyValue(COMPACTION_START, 7, 11, "/t/f2"), + createKeyValue(MUTATION, 8, 11, m4)}; + + Arrays.sort(entries1); + + Map logs = new TreeMap<>(); + logs.put("entries1", entries1); + + List mutations1 = recover(logs, e1); + Assert.assertEquals(1, mutations1.size()); + Assert.assertEquals(m2, mutations1.get(0)); + + List mutations2 = recover(logs, e2); + Assert.assertEquals(2, mutations2.size()); + Assert.assertEquals(m3, mutations2.get(0)); + Assert.assertEquals(m4, mutations2.get(1)); + + KeyValue entries2[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"), + createKeyValue(DEFINE_TABLET, 9, 11, e2), createKeyValue(COMPACTION_FINISH, 10, 11, null)}; + Arrays.sort(entries2); + logs.put("entries2", entries2); + + mutations2 = recover(logs, e2); + Assert.assertEquals(1, mutations2.size()); + Assert.assertEquals(m4, mutations2.get(0)); + } + private void runPathTest(boolean startMatches, String compactionStartFile, String... tabletFiles) throws IOException { Mutation m1 = new ServerMutation(new Text("row1")); diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/logger/LogFileTest.java index 2e8d220,5e26a9d..22c83cc --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/logger/LogFileTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/logger/LogFileTest.java @@@ -80,9 -79,9 +80,9 @@@ public class LogFileTest readWrite(COMPACTION_START, 3, 4, "some file", null, null, key, value); assertEquals(key.event, COMPACTION_START); assertEquals(key.seq, 3); - assertEquals(key.tid, 4); + assertEquals(key.tabletId, 4); assertEquals(key.filename, "some file"); - KeyExtent tablet = new KeyExtent("table", new Text("bbbb"), new Text("aaaa")); + KeyExtent tablet = new KeyExtent(Table.ID.of("table"), new Text("bbbb"), new Text("aaaa")); readWrite(DEFINE_TABLET, 5, 6, null, tablet, null, key, value); assertEquals(key.event, DEFINE_TABLET); assertEquals(key.seq, 5); diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystemTest.java index b270428,107060b..25cf68b --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystemTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystemTest.java @@@ -78,9 -78,9 +78,8 @@@ public class AccumuloReplicaSystemTest * are solely for testing that each LogEvents is handled, order is not important. */ key.event = LogEvents.DEFINE_TABLET; - key.tablet = new KeyExtent("1", null, null); + key.tablet = new KeyExtent(Table.ID.of("1"), null, null); - key.tid = 1; - + key.tabletId = 1; - key.write(dos); value.write(dos); @@@ -93,8 -93,8 +92,8 @@@ value.write(dos); key.event = LogEvents.DEFINE_TABLET; - key.tablet = new KeyExtent("2", null, null); + key.tablet = new KeyExtent(Table.ID.of("2"), null, null); - key.tid = 2; + key.tabletId = 2; value.mutations = Collections.emptyList(); key.write(dos); @@@ -124,8 -124,8 +123,8 @@@ value.write(dos); key.event = LogEvents.DEFINE_TABLET; - key.tablet = new KeyExtent("1", null, null); + key.tablet = new KeyExtent(Table.ID.of("1"), null, null); - key.tid = 3; + key.tabletId = 3; value.mutations = Collections.emptyList(); key.write(dos); @@@ -140,9 -140,9 +139,9 @@@ key.tablet = null; key.event = LogEvents.MUTATION; - key.tid = 3; + key.tabletId = 3; key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID(); - value.mutations = Arrays. asList(new ServerMutation(new Text("row"))); + value.mutations = Arrays.asList(new ServerMutation(new Text("row"))); key.write(dos); value.write(dos); @@@ -186,8 -187,8 +185,8 @@@ * are solely for testing that each LogEvents is handled, order is not important. */ key.event = LogEvents.DEFINE_TABLET; - key.tablet = new KeyExtent("1", null, null); + key.tablet = new KeyExtent(Table.ID.of("1"), null, null); - key.tid = 1; + key.tabletId = 1; key.write(dos); value.write(dos); @@@ -201,8 -202,8 +200,8 @@@ value.write(dos); key.event = LogEvents.DEFINE_TABLET; - key.tablet = new KeyExtent("2", null, null); + key.tablet = new KeyExtent(Table.ID.of("2"), null, null); - key.tid = 2; + key.tabletId = 2; value.mutations = Collections.emptyList(); key.write(dos); @@@ -232,8 -233,8 +231,8 @@@ value.write(dos); key.event = LogEvents.DEFINE_TABLET; - key.tablet = new KeyExtent("1", null, null); + key.tablet = new KeyExtent(Table.ID.of("1"), null, null); - key.tid = 3; + key.tabletId = 3; value.mutations = Collections.emptyList(); key.write(dos); @@@ -248,9 -249,9 +247,9 @@@ key.tablet = null; key.event = LogEvents.MUTATION; - key.tid = 3; + key.tabletId = 3; key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID(); - value.mutations = Arrays. asList(new ServerMutation(new Text("row"))); + value.mutations = Arrays.asList(new ServerMutation(new Text("row"))); key.write(dos); value.write(dos); @@@ -392,8 -395,8 +391,8 @@@ * are solely for testing that each LogEvents is handled, order is not important. */ key.event = LogEvents.DEFINE_TABLET; - key.tablet = new KeyExtent("1", null, null); + key.tablet = new KeyExtent(Table.ID.of("1"), null, null); - key.tid = 1; + key.tabletId = 1; key.write(dos); value.write(dos); @@@ -408,9 -411,9 +407,9 @@@ key.tablet = null; key.event = LogEvents.MUTATION; - key.tid = 1; + key.tabletId = 1; key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID(); - value.mutations = Arrays. asList(new ServerMutation(new Text("row"))); + value.mutations = Arrays.asList(new ServerMutation(new Text("row"))); key.write(dos); value.write(dos); diff --cc test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java index a2025bf,bdde3f7..1146c8e --- a/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java +++ b/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java @@@ -117,9 -116,9 +117,9 @@@ public class UnusedWalDoesntCloseReplic value.write(out); key.event = LogEvents.DEFINE_TABLET; - key.tablet = new KeyExtent(Integer.toString(fakeTableId), null, null); - key.seq = 1l; + key.tablet = new KeyExtent(Table.ID.of(Integer.toString(fakeTableId)), null, null); + key.seq = 1L; - key.tid = 1; + key.tabletId = 1; key.write(dos); value.write(dos); -- To stop receiving notification emails like this one, please contact kturner@apache.org.