Return-Path: X-Original-To: apmail-accumulo-commits-archive@www.apache.org Delivered-To: apmail-accumulo-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 25E4418DB6 for ; Wed, 3 Feb 2016 03:09:00 +0000 (UTC) Received: (qmail 5485 invoked by uid 500); 3 Feb 2016 03:08:59 -0000 Delivered-To: apmail-accumulo-commits-archive@accumulo.apache.org Received: (qmail 5342 invoked by uid 500); 3 Feb 2016 03:08:59 -0000 Mailing-List: contact commits-help@accumulo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@accumulo.apache.org Delivered-To: mailing list commits@accumulo.apache.org Received: (qmail 5223 invoked by uid 99); 3 Feb 2016 03:08:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Feb 2016 03:08:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 828BCDFD7D; Wed, 3 Feb 2016 03:08:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ctubbsii@apache.org To: commits@accumulo.apache.org Date: Wed, 03 Feb 2016 03:09:02 -0000 Message-Id: <34e5d25475cb41358a4c2b3fe5bf3f92@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [04/21] accumulo git commit: Merge branch '1.6' into 1.7 Merge branch '1.6' into 1.7 Conflicts: core/src/main/java/org/apache/accumulo/core/util/Merge.java Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/84d3e5c4 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/84d3e5c4 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/84d3e5c4 Branch: refs/heads/1.7 Commit: 84d3e5c4fa3064b644651d9ab958cb499f02c751 Parents: 6cc5ab5 c462001 Author: Christopher Tubbs Authored: Tue Feb 2 19:19:26 2016 -0500 Committer: Christopher Tubbs Committed: Tue Feb 2 19:19:26 2016 -0500 ---------------------------------------------------------------------- .../core/metadata/schema/DataFileValue.java | 18 ++++++++++++--- .../org/apache/accumulo/core/util/Merge.java | 8 ++----- .../apache/accumulo/server/init/Initialize.java | 8 ++++--- .../constraints/MetadataConstraintsTest.java | 17 +++++++------- .../iterators/MetadataBulkLoadFilterTest.java | 5 ++-- .../apache/accumulo/server/util/CloneTest.java | 24 ++++++++++++++------ .../metadata/MetadataBatchScanTest.java | 3 ++- 7 files changed, 53 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3e5c4/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java index cebe041,dfa1114..5f1379d --- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java +++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java @@@ -16,6 -16,10 +16,10 @@@ */ package org.apache.accumulo.core.metadata.schema; -import static com.google.common.base.Charsets.UTF_8; ++import static java.nio.charset.StandardCharsets.UTF_8; + + import org.apache.accumulo.core.data.Value; + public class DataFileValue { private long size; private long numEntries; http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3e5c4/core/src/main/java/org/apache/accumulo/core/util/Merge.java ---------------------------------------------------------------------- diff --cc core/src/main/java/org/apache/accumulo/core/util/Merge.java index 3c60d25,c5e3e8e..9f6f6ab --- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java +++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java @@@ -31,9 -29,10 +29,10 @@@ import org.apache.accumulo.core.conf.Ac import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.data.impl.KeyExtent; import org.apache.accumulo.core.metadata.MetadataTable; + import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.security.Authorizations; http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3e5c4/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java ---------------------------------------------------------------------- diff --cc server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java index 9b9dac9,491dc25..4e5864e --- a/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java +++ b/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java @@@ -30,12 -27,9 +30,10 @@@ import java.util.HashMap import java.util.HashSet; import java.util.Locale; import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; import java.util.UUID; --import jline.console.ConsoleReader; - import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.cli.Help; import org.apache.accumulo.core.client.AccumuloSecurityException; @@@ -58,21 -48,9 +56,22 @@@ import org.apache.accumulo.core.master. import org.apache.accumulo.core.master.thrift.MasterGoalState; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; ++import org.apache.accumulo.core.metadata.schema.DataFileValue; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.CurrentLocationColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.FutureLocationColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily; +import org.apache.accumulo.core.replication.ReplicationConstants; +import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection; +import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection; +import org.apache.accumulo.core.replication.ReplicationTable; import org.apache.accumulo.core.util.CachedConfiguration; import org.apache.accumulo.core.util.ColumnFQ; +import org.apache.accumulo.core.util.LocalityGroupUtil; import org.apache.accumulo.core.util.Pair; import org.apache.accumulo.core.volume.VolumeConfiguration; import org.apache.accumulo.core.zookeeper.ZooUtil; @@@ -104,17 -77,12 +103,19 @@@ import org.apache.hadoop.fs.FileStatus import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; -import org.apache.log4j.Logger; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooDefs.Ids; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.beust.jcommander.Parameter; +import com.google.auto.service.AutoService; +import com.google.common.base.Joiner; +import com.google.common.base.Optional; + ++import jline.console.ConsoleReader; + /** * This class is used to setup the directory structure and the root tablet to get an instance started * @@@ -154,8 -121,7 +155,9 @@@ public class Initialize implements Keyw } private static HashMap initialMetadataConf = new HashMap(); + private static HashMap initialMetadataCombinerConf = new HashMap(); + private static HashMap initialReplicationTableConf = new HashMap(); + static { initialMetadataConf.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "32K"); initialMetadataConf.put(Property.TABLE_FILE_REPLICATION.getKey(), "5"); @@@ -472,19 -342,16 +474,19 @@@ tabletWriter.close(); } - private static void createEntriesForTablet(FileSKVWriter writer, String tableId, String tabletDir, Text tabletPrevEndRow, Text tabletEndRow) - throws IOException { - Text extent = new Text(KeyExtent.getMetadataEntry(new Text(tableId), tabletEndRow)); - addEntry(writer, extent, DIRECTORY_COLUMN, new Value(tabletDir.getBytes(UTF_8))); - addEntry(writer, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8))); - addEntry(writer, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tabletPrevEndRow)); + private static void createEntriesForTablet(TreeMap map, Tablet tablet) { - Value EMPTY_SIZE = new Value("0,0".getBytes(UTF_8)); ++ Value EMPTY_SIZE = new DataFileValue(0, 0).encodeAsValue(); + Text extent = new Text(KeyExtent.getMetadataEntry(new Text(tablet.tableId), tablet.endRow)); + addEntry(map, extent, DIRECTORY_COLUMN, new Value(tablet.dir.getBytes(UTF_8))); + addEntry(map, extent, TIME_COLUMN, new Value((TabletTime.LOGICAL_TIME_ID + "0").getBytes(UTF_8))); + addEntry(map, extent, PREV_ROW_COLUMN, KeyExtent.encodePrevEndRow(tablet.prevEndRow)); + for (String file : tablet.files) { + addEntry(map, extent, new ColumnFQ(DataFileColumnFamily.NAME, new Text(file)), EMPTY_SIZE); + } } - private static void addEntry(FileSKVWriter writer, Text row, ColumnFQ col, Value value) throws IOException { - writer.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier(), 0), value); + private static void addEntry(TreeMap map, Text row, ColumnFQ col, Value value) { + map.put(new Key(row, col.getColumnFamily(), col.getColumnQualifier(), 0), value); } private static void createDirectories(VolumeManager fs, String... dirs) throws IOException { http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3e5c4/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java ---------------------------------------------------------------------- diff --cc server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java index 7b6eec2,7b6eec2..61167fd --- a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java @@@ -25,6 -25,6 +25,7 @@@ import java.util.List import org.apache.accumulo.core.conf.AccumuloConfiguration; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; ++import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.fate.zookeeper.TransactionWatcher.Arbitrator; @@@ -140,7 -140,7 +141,7 @@@ public class MetadataConstraintsTest // inactive txid m = new Mutation(new Text("0;foo")); m.put(TabletsSection.BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("12345".getBytes())); -- m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new Value("1,1".getBytes())); ++ m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new DataFileValue(1, 1).encodeAsValue()); violations = mc.check(null, m); assertNotNull(violations); assertEquals(1, violations.size()); @@@ -149,7 -149,7 +150,7 @@@ // txid that throws exception m = new Mutation(new Text("0;foo")); m.put(TabletsSection.BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("9".getBytes())); -- m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new Value("1,1".getBytes())); ++ m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new DataFileValue(1, 1).encodeAsValue()); violations = mc.check(null, m); assertNotNull(violations); assertEquals(1, violations.size()); @@@ -158,7 -158,7 +159,7 @@@ // active txid w/ file m = new Mutation(new Text("0;foo")); m.put(TabletsSection.BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5".getBytes())); -- m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new Value("1,1".getBytes())); ++ m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new DataFileValue(1, 1).encodeAsValue()); violations = mc.check(null, m); assertNull(violations); @@@ -173,9 -173,9 +174,9 @@@ // two active txids w/ files m = new Mutation(new Text("0;foo")); m.put(TabletsSection.BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5".getBytes())); -- m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new Value("1,1".getBytes())); ++ m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new DataFileValue(1, 1).encodeAsValue()); m.put(TabletsSection.BulkFileColumnFamily.NAME, new Text("/someFile2"), new Value("7".getBytes())); -- m.put(DataFileColumnFamily.NAME, new Text("/someFile2"), new Value("1,1".getBytes())); ++ m.put(DataFileColumnFamily.NAME, new Text("/someFile2"), new DataFileValue(1, 1).encodeAsValue()); violations = mc.check(null, m); assertNotNull(violations); assertEquals(1, violations.size()); @@@ -184,16 -184,16 +185,16 @@@ // two files w/ one active txid m = new Mutation(new Text("0;foo")); m.put(TabletsSection.BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5".getBytes())); -- m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new Value("1,1".getBytes())); ++ m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new DataFileValue(1, 1).encodeAsValue()); m.put(TabletsSection.BulkFileColumnFamily.NAME, new Text("/someFile2"), new Value("5".getBytes())); -- m.put(DataFileColumnFamily.NAME, new Text("/someFile2"), new Value("1,1".getBytes())); ++ m.put(DataFileColumnFamily.NAME, new Text("/someFile2"), new DataFileValue(1, 1).encodeAsValue()); violations = mc.check(null, m); assertNull(violations); // two loaded w/ one active txid and one file m = new Mutation(new Text("0;foo")); m.put(TabletsSection.BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5".getBytes())); -- m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new Value("1,1".getBytes())); ++ m.put(DataFileColumnFamily.NAME, new Text("/someFile"), new DataFileValue(1, 1).encodeAsValue()); m.put(TabletsSection.BulkFileColumnFamily.NAME, new Text("/someFile2"), new Value("5".getBytes())); violations = mc.check(null, m); assertNotNull(violations); http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3e5c4/server/base/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java ---------------------------------------------------------------------- diff --cc server/base/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java index 7e9543f,cfdd5e9..ed662a5 --- a/server/base/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java @@@ -30,9 -30,8 +30,10 @@@ import org.apache.accumulo.core.iterato import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.SortedMapIterator; ++import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; +import org.apache.accumulo.core.security.Authorizations; import org.apache.accumulo.core.util.ColumnFQ; import org.apache.accumulo.fate.zookeeper.TransactionWatcher.Arbitrator; import org.apache.hadoop.io.Text; @@@ -85,12 -84,12 +86,12 @@@ public class MetadataBulkLoadFilterTes // following should not be deleted by filter put(tm1, "2;m", TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN, "/t1"); -- put(tm1, "2;m", DataFileColumnFamily.NAME, "/t1/file1", "1,1"); ++ put(tm1, "2;m", DataFileColumnFamily.NAME, "/t1/file1", new DataFileValue(1, 1).encodeAsString()); put(tm1, "2;m", TabletsSection.BulkFileColumnFamily.NAME, "/t1/file1", "5"); put(tm1, "2;m", TabletsSection.BulkFileColumnFamily.NAME, "/t1/file3", "7"); put(tm1, "2;m", TabletsSection.BulkFileColumnFamily.NAME, "/t1/file4", "9"); put(tm1, "2<", TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN, "/t2"); -- put(tm1, "2<", DataFileColumnFamily.NAME, "/t2/file2", "1,1"); ++ put(tm1, "2<", DataFileColumnFamily.NAME, "/t2/file2", new DataFileValue(1, 1).encodeAsString()); put(tm1, "2<", TabletsSection.BulkFileColumnFamily.NAME, "/t2/file6", "5"); put(tm1, "2<", TabletsSection.BulkFileColumnFamily.NAME, "/t2/file7", "7"); put(tm1, "2<", TabletsSection.BulkFileColumnFamily.NAME, "/t2/file8", "9"); http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3e5c4/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java ---------------------------------------------------------------------- diff --cc server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java index 0c696a1,74d5e64..9d33935 --- a/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java @@@ -28,10 -29,11 +29,11 @@@ import org.apache.accumulo.core.client. import org.apache.accumulo.core.client.mock.MockInstance; import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.KeyExtent; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.data.impl.KeyExtent; import org.apache.accumulo.core.metadata.MetadataTable; + import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.security.Authorizations; http://git-wip-us.apache.org/repos/asf/accumulo/blob/84d3e5c4/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java ---------------------------------------------------------------------- diff --cc test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java index 620ae8b,07bfdc7..1c7ce67 --- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java @@@ -37,8 -36,10 +37,9 @@@ import org.apache.accumulo.core.client. import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.data.impl.KeyExtent; import org.apache.accumulo.core.metadata.MetadataTable; + import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.security.Authorizations;