accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [11/21] accumulo git commit: Merge branch '1.7'
Date Wed, 03 Feb 2016 03:09:09 GMT
Merge branch '1.7'


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1c696a90
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1c696a90
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1c696a90

Branch: refs/heads/master
Commit: 1c696a90520c0ba21d19007dfb2311b52052be48
Parents: b4cd908 668c340
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Tue Feb 2 20:15:45 2016 -0500
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Tue Feb 2 20:15:45 2016 -0500

----------------------------------------------------------------------
 .../core/metadata/schema/DataFileValue.java       | 18 +++++++++++++++---
 .../java/org/apache/accumulo/core/util/Merge.java |  8 ++------
 pom.xml                                           |  2 +-
 .../apache/accumulo/server/init/Initialize.java   |  8 +++++---
 .../constraints/MetadataConstraintsTest.java      | 17 +++++++++--------
 .../iterators/MetadataBulkLoadFilterTest.java     |  5 +++--
 .../java/org/apache/accumulo/test/CloneIT.java    |  9 +++++----
 .../metadata/MetadataBatchScanTest.java           |  3 ++-
 8 files changed, 42 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/1c696a90/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/1c696a90/server/base/src/main/java/org/apache/accumulo/server/init/Initialize.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/accumulo/blob/1c696a90/server/base/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
index 1b30530,ed662a5..04bcbde
--- a/server/base/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/iterators/MetadataBulkLoadFilterTest.java
@@@ -26,10 -26,14 +26,11 @@@ import org.apache.accumulo.core.data.By
  import org.apache.accumulo.core.data.Key;
  import org.apache.accumulo.core.data.Range;
  import org.apache.accumulo.core.data.Value;
 -import org.apache.accumulo.core.iterators.IteratorEnvironment;
  import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
 -import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
  import org.apache.accumulo.core.iterators.SortedMapIterator;
+ import org.apache.accumulo.core.metadata.schema.DataFileValue;
  import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
  import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 -import org.apache.accumulo.core.security.Authorizations;
  import org.apache.accumulo.core.util.ColumnFQ;
  import org.apache.accumulo.fate.zookeeper.TransactionWatcher.Arbitrator;
  import org.apache.hadoop.io.Text;

http://git-wip-us.apache.org/repos/asf/accumulo/blob/1c696a90/test/src/main/java/org/apache/accumulo/test/CloneIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/CloneIT.java
index e292b75,0000000..441e924
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/CloneIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/CloneIT.java
@@@ -1,390 -1,0 +1,391 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.accumulo.test;
 +
 +import static org.junit.Assert.assertEquals;
 +import static org.junit.Assert.assertTrue;
 +
 +import java.util.HashSet;
 +import java.util.Map.Entry;
 +
 +import org.apache.accumulo.core.client.BatchWriter;
 +import org.apache.accumulo.core.client.BatchWriterConfig;
 +import org.apache.accumulo.core.client.Connector;
 +import org.apache.accumulo.core.client.Scanner;
 +import org.apache.accumulo.core.data.Key;
 +import org.apache.accumulo.core.data.Mutation;
 +import org.apache.accumulo.core.data.Value;
 +import org.apache.accumulo.core.data.impl.KeyExtent;
++import org.apache.accumulo.core.metadata.schema.DataFileValue;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 +import org.apache.accumulo.core.security.Authorizations;
 +import org.apache.accumulo.harness.AccumuloClusterHarness;
 +import org.apache.accumulo.server.util.MetadataTableUtil;
 +import org.apache.accumulo.server.util.TabletIterator;
 +import org.apache.hadoop.io.Text;
 +import org.junit.Test;
 +
 +public class CloneIT extends AccumuloClusterHarness {
 +
 +  @Test
 +  public void testNoFiles() throws Exception {
 +    Connector conn = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    conn.tableOperations().create(tableName);
 +
 +    KeyExtent ke = new KeyExtent(new Text("0"), null, null);
 +    Mutation mut = ke.getPrevRowUpdateMutation();
 +
 +    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0".getBytes()));
 +    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value("/default_tablet".getBytes()));
 +
 +    BatchWriter bw1 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    bw1.addMutation(mut);
 +
 +    bw1.close();
 +
 +    BatchWriter bw2 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    MetadataTableUtil.initializeClone(tableName, "0", "1", conn, bw2);
 +
 +    int rc = MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +
 +    assertEquals(0, rc);
 +
 +    // scan tables metadata entries and confirm the same
 +
 +  }
 +
 +  @Test
 +  public void testFilesChange() throws Exception {
 +    Connector conn = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    conn.tableOperations().create(tableName);
 +
 +    KeyExtent ke = new KeyExtent(new Text("0"), null, null);
 +    Mutation mut = ke.getPrevRowUpdateMutation();
 +
 +    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0".getBytes()));
 +    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value("/default_tablet".getBytes()));
-     mut.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf", "1,200");
++    mut.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf", new DataFileValue(1,
200).encodeAsString());
 +
 +    BatchWriter bw1 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    bw1.addMutation(mut);
 +
 +    bw1.flush();
 +
 +    BatchWriter bw2 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    MetadataTableUtil.initializeClone(tableName, "0", "1", conn, bw2);
 +
 +    Mutation mut2 = new Mutation(ke.getMetadataEntry());
 +    mut2.putDelete(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf");
-     mut2.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/1_0.rf", "2,300");
++    mut2.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/1_0.rf", new DataFileValue(2,
300).encodeAsString());
 +
 +    bw1.addMutation(mut2);
 +    bw1.flush();
 +
 +    int rc = MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +
 +    assertEquals(1, rc);
 +
 +    rc = MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +
 +    assertEquals(0, rc);
 +
 +    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
 +    scanner.setRange(new KeyExtent(new Text("1"), null, null).toMetadataRange());
 +
 +    HashSet<String> files = new HashSet<String>();
 +
 +    for (Entry<Key,Value> entry : scanner) {
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME))
 +        files.add(entry.getKey().getColumnQualifier().toString());
 +    }
 +
 +    assertEquals(1, files.size());
 +    assertTrue(files.contains("../0/default_tablet/1_0.rf"));
 +
 +  }
 +
 +  // test split where files of children are the same
 +  @Test
 +  public void testSplit1() throws Exception {
 +    Connector conn = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    conn.tableOperations().create(tableName);
 +
 +    BatchWriter bw1 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    bw1.addMutation(createTablet("0", null, null, "/default_tablet", "/default_tablet/0_0.rf"));
 +
 +    bw1.flush();
 +
 +    BatchWriter bw2 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    MetadataTableUtil.initializeClone(tableName, "0", "1", conn, bw2);
 +
 +    bw1.addMutation(createTablet("0", "m", null, "/default_tablet", "/default_tablet/0_0.rf"));
 +    bw1.addMutation(createTablet("0", null, "m", "/t-1", "/default_tablet/0_0.rf"));
 +
 +    bw1.flush();
 +
 +    int rc = MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +
 +    assertEquals(0, rc);
 +
 +    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
 +    scanner.setRange(new KeyExtent(new Text("1"), null, null).toMetadataRange());
 +
 +    HashSet<String> files = new HashSet<String>();
 +
 +    int count = 0;
 +    for (Entry<Key,Value> entry : scanner) {
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        files.add(entry.getKey().getColumnQualifier().toString());
 +        count++;
 +      }
 +    }
 +
 +    assertEquals(1, count);
 +    assertEquals(1, files.size());
 +    assertTrue(files.contains("../0/default_tablet/0_0.rf"));
 +  }
 +
 +  // test split where files of children differ... like majc and split occurred
 +  @Test
 +  public void testSplit2() throws Exception {
 +    Connector conn = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    conn.tableOperations().create(tableName);
 +
 +    BatchWriter bw1 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    bw1.addMutation(createTablet("0", null, null, "/default_tablet", "/default_tablet/0_0.rf"));
 +
 +    bw1.flush();
 +
 +    BatchWriter bw2 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    MetadataTableUtil.initializeClone(tableName, "0", "1", conn, bw2);
 +
 +    bw1.addMutation(createTablet("0", "m", null, "/default_tablet", "/default_tablet/1_0.rf"));
 +    Mutation mut3 = createTablet("0", null, "m", "/t-1", "/default_tablet/1_0.rf");
 +    mut3.putDelete(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf");
 +    bw1.addMutation(mut3);
 +
 +    bw1.flush();
 +
 +    int rc = MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +
 +    assertEquals(1, rc);
 +
 +    rc = MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +
 +    assertEquals(0, rc);
 +
 +    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
 +    scanner.setRange(new KeyExtent(new Text("1"), null, null).toMetadataRange());
 +
 +    HashSet<String> files = new HashSet<String>();
 +
 +    int count = 0;
 +
 +    for (Entry<Key,Value> entry : scanner) {
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        files.add(entry.getKey().getColumnQualifier().toString());
 +        count++;
 +      }
 +    }
 +
 +    assertEquals(1, files.size());
 +    assertEquals(2, count);
 +    assertTrue(files.contains("../0/default_tablet/1_0.rf"));
 +  }
 +
 +  private static Mutation deleteTablet(String tid, String endRow, String prevRow, String
dir, String file) throws Exception {
 +    KeyExtent ke = new KeyExtent(new Text(tid), endRow == null ? null : new Text(endRow),
prevRow == null ? null : new Text(prevRow));
 +    Mutation mut = new Mutation(ke.getMetadataEntry());
 +    TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.putDelete(mut);
 +    TabletsSection.ServerColumnFamily.TIME_COLUMN.putDelete(mut);
 +    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.putDelete(mut);
 +    mut.putDelete(DataFileColumnFamily.NAME.toString(), file);
 +
 +    return mut;
 +  }
 +
 +  private static Mutation createTablet(String tid, String endRow, String prevRow, String
dir, String file) throws Exception {
 +    KeyExtent ke = new KeyExtent(new Text(tid), endRow == null ? null : new Text(endRow),
prevRow == null ? null : new Text(prevRow));
 +    Mutation mut = ke.getPrevRowUpdateMutation();
 +
 +    TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0".getBytes()));
 +    TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes()));
-     mut.put(DataFileColumnFamily.NAME.toString(), file, "10,200");
++    mut.put(DataFileColumnFamily.NAME.toString(), file, new DataFileValue(10, 200).encodeAsString());
 +
 +    return mut;
 +  }
 +
 +  // test two tablets splitting into four
 +  @Test
 +  public void testSplit3() throws Exception {
 +    Connector conn = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    conn.tableOperations().create(tableName);
 +
 +    BatchWriter bw1 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    bw1.addMutation(createTablet("0", "m", null, "/d1", "/d1/file1"));
 +    bw1.addMutation(createTablet("0", null, "m", "/d2", "/d2/file2"));
 +
 +    bw1.flush();
 +
 +    BatchWriter bw2 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    MetadataTableUtil.initializeClone(tableName, "0", "1", conn, bw2);
 +
 +    bw1.addMutation(createTablet("0", "f", null, "/d1", "/d1/file3"));
 +    bw1.addMutation(createTablet("0", "m", "f", "/d3", "/d1/file1"));
 +    bw1.addMutation(createTablet("0", "s", "m", "/d2", "/d2/file2"));
 +    bw1.addMutation(createTablet("0", null, "s", "/d4", "/d2/file2"));
 +
 +    bw1.flush();
 +
 +    int rc = MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +
 +    assertEquals(0, rc);
 +
 +    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
 +    scanner.setRange(new KeyExtent(new Text("1"), null, null).toMetadataRange());
 +
 +    HashSet<String> files = new HashSet<String>();
 +
 +    int count = 0;
 +    for (Entry<Key,Value> entry : scanner) {
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        files.add(entry.getKey().getColumnQualifier().toString());
 +        count++;
 +      }
 +    }
 +
 +    assertEquals(2, count);
 +    assertEquals(2, files.size());
 +    assertTrue(files.contains("../0/d1/file1"));
 +    assertTrue(files.contains("../0/d2/file2"));
 +  }
 +
 +  // test cloned marker
 +  @Test
 +  public void testClonedMarker() throws Exception {
 +    Connector conn = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    conn.tableOperations().create(tableName);
 +
 +    BatchWriter bw1 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    bw1.addMutation(createTablet("0", "m", null, "/d1", "/d1/file1"));
 +    bw1.addMutation(createTablet("0", null, "m", "/d2", "/d2/file2"));
 +
 +    bw1.flush();
 +
 +    BatchWriter bw2 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    MetadataTableUtil.initializeClone(tableName, "0", "1", conn, bw2);
 +
 +    bw1.addMutation(deleteTablet("0", "m", null, "/d1", "/d1/file1"));
 +    bw1.addMutation(deleteTablet("0", null, "m", "/d2", "/d2/file2"));
 +
 +    bw1.flush();
 +
 +    bw1.addMutation(createTablet("0", "f", null, "/d1", "/d1/file3"));
 +    bw1.addMutation(createTablet("0", "m", "f", "/d3", "/d1/file1"));
 +    bw1.addMutation(createTablet("0", "s", "m", "/d2", "/d2/file3"));
 +    bw1.addMutation(createTablet("0", null, "s", "/d4", "/d4/file3"));
 +
 +    bw1.flush();
 +
 +    int rc = MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +
 +    assertEquals(1, rc);
 +
 +    bw1.addMutation(deleteTablet("0", "m", "f", "/d3", "/d1/file1"));
 +
 +    bw1.flush();
 +
 +    bw1.addMutation(createTablet("0", "m", "f", "/d3", "/d1/file3"));
 +
 +    bw1.flush();
 +
 +    rc = MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +
 +    assertEquals(0, rc);
 +
 +    Scanner scanner = conn.createScanner(tableName, Authorizations.EMPTY);
 +    scanner.setRange(new KeyExtent(new Text("1"), null, null).toMetadataRange());
 +
 +    HashSet<String> files = new HashSet<String>();
 +
 +    int count = 0;
 +    for (Entry<Key,Value> entry : scanner) {
 +      if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) {
 +        files.add(entry.getKey().getColumnQualifier().toString());
 +        count++;
 +      }
 +    }
 +
 +    assertEquals(3, count);
 +    assertEquals(3, files.size());
 +    assertTrue(files.contains("../0/d1/file1"));
 +    assertTrue(files.contains("../0/d2/file3"));
 +    assertTrue(files.contains("../0/d4/file3"));
 +  }
 +
 +  // test two tablets splitting into four
 +  @Test
 +  public void testMerge() throws Exception {
 +    Connector conn = getConnector();
 +    String tableName = getUniqueNames(1)[0];
 +    conn.tableOperations().create(tableName);
 +
 +    BatchWriter bw1 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    bw1.addMutation(createTablet("0", "m", null, "/d1", "/d1/file1"));
 +    bw1.addMutation(createTablet("0", null, "m", "/d2", "/d2/file2"));
 +
 +    bw1.flush();
 +
 +    BatchWriter bw2 = conn.createBatchWriter(tableName, new BatchWriterConfig());
 +
 +    MetadataTableUtil.initializeClone(tableName, "0", "1", conn, bw2);
 +
 +    bw1.addMutation(deleteTablet("0", "m", null, "/d1", "/d1/file1"));
 +    Mutation mut = createTablet("0", null, null, "/d2", "/d2/file2");
-     mut.put(DataFileColumnFamily.NAME.toString(), "/d1/file1", "10,200");
++    mut.put(DataFileColumnFamily.NAME.toString(), "/d1/file1", new DataFileValue(10, 200).encodeAsString());
 +    bw1.addMutation(mut);
 +
 +    bw1.flush();
 +
 +    try {
 +      MetadataTableUtil.checkClone(tableName, "0", "1", conn, bw2);
 +      assertTrue(false);
 +    } catch (TabletIterator.TabletDeletedException tde) {}
 +
 +  }
 +
 +}


Mime
View raw message