accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [01/19] accumulo git commit: ACCUMULO-4165 Made RFileWriter start default LG automatically
Date Fri, 03 Jun 2016 02:21:00 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.6 63a8a5d7e -> beb69cdfc
  refs/heads/1.7 d33b2a09d -> 0eab0ecff
  refs/heads/1.8 24465d675 -> 68576ad59
  refs/heads/master c77648377 -> 35de95264


ACCUMULO-4165 Made RFileWriter start default LG automatically


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/24465d67
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/24465d67
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/24465d67

Branch: refs/heads/master
Commit: 24465d675501440685aa026a032211afa837db64
Parents: e67317c
Author: Keith Turner <keith@deenlo.com>
Authored: Thu Jun 2 16:57:48 2016 -0400
Committer: Keith Turner <keith@deenlo.com>
Committed: Thu Jun 2 16:57:48 2016 -0400

----------------------------------------------------------------------
 .../accumulo/core/client/rfile/RFileWriter.java | 29 +++++++++---------
 .../accumulo/core/client/rfile/RFileTest.java   | 31 ++++----------------
 2 files changed, 20 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/24465d67/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
index aad4908..13da017 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileWriter.java
@@ -42,10 +42,9 @@ import com.google.common.base.Preconditions;
  * {@link TableOperations#importDirectory(String, String, String, boolean)}
  *
  * <p>
- * A RFileWriter has the following constraints. Violating these contraints will result in
runtime exceptions.
+ * A RFileWriter has the following constraints. Violating these constraints will result in
runtime exceptions.
  *
  * <ul>
- * <li>Before appending any keys, a locality group must be started by calling one of
the startNewLocalityGroup functions or startDefaultLocalityGroup.</li>
  * <li>Keys must be appended in sorted order within a locality group.</li>
  * <li>Locality groups must have a mutually exclusive set of column families.</li>
  * <li>The default locality group must be started last.</li>
@@ -128,14 +127,17 @@ public class RFileWriter implements AutoCloseable {
   }
 
   /**
-   * See have doc for {@link #startNewLocalityGroup(String, List)}
+   * See javadoc for {@link #startNewLocalityGroup(String, List)}
+   *
+   * @throws IllegalStateException
+   *           When default locality group already started.
    */
   public void startNewLocalityGroup(String name, byte[]... families) throws IOException {
     startNewLocalityGroup(name, Arrays.asList(families));
   }
 
   /**
-   * See have doc for {@link #startNewLocalityGroup(String, List)}.
+   * See javadoc for {@link #startNewLocalityGroup(String, List)}.
    *
    * @param families
    *          will be encoded using UTF-8
@@ -152,7 +154,7 @@ public class RFileWriter implements AutoCloseable {
   }
 
   /**
-   * See have doc for {@link #startNewLocalityGroup(String, List)}.
+   * See javadoc for {@link #startNewLocalityGroup(String, List)}.
    *
    * @param families
    *          will be encoded using UTF-8
@@ -170,7 +172,7 @@ public class RFileWriter implements AutoCloseable {
 
   /**
    * A locality group in which the column families do not need to specified. The locality
group must be started after all other locality groups. Can not append
-   * column families that were in a previous locality group.
+   * column families that were in a previous locality group. If no locality groups were started,
then the first append will start the default locality group.
    *
    * @throws IllegalStateException
    *           When default locality group already started.
@@ -184,7 +186,8 @@ public class RFileWriter implements AutoCloseable {
   }
 
   /**
-   * Append the key and value to the last locality group that was started.
+   * Append the key and value to the last locality group that was started. If no locality
group was started, then the default group will automatically be
+   * started.
    *
    * @param key
    *          This key must be greater than or equal to the last key appended. For non-default
locality groups, the keys column family must be one of the column
@@ -195,11 +198,11 @@ public class RFileWriter implements AutoCloseable {
    * @throws IllegalArgumentException
    *           This is thrown when data is appended out of order OR when the key contains
a invalid visibility OR when a column family is not valid for a
    *           locality group.
-   * @throws IllegalStateException
-   *           Thrown when no locality group was started.
    */
   public void append(Key key, Value val) throws IOException {
-    Preconditions.checkState(startedLG, "No locality group was started");
+    if (!startedLG) {
+      startDefaultLocalityGroup();
+    }
     Boolean wasChecked = (Boolean) validVisibilities.get(key.getColumnVisibilityData());
     if (wasChecked == null) {
       byte[] cv = key.getColumnVisibilityData().toArray();
@@ -214,16 +217,14 @@ public class RFileWriter implements AutoCloseable {
    *
    * @param keyValues
    *          The keys must be in sorted order. The first key returned by the iterable must
be greater than or equal to the last key appended. For non-default
-   *          locality groups, the keys column family must be one of the column families
specified when calling startNewLocalityGroup(). Must be non-null.
+   *          locality groups, the keys column family must be one of the column families
specified when calling startNewLocalityGroup(). Must be non-null. If no
+   *          locality group was started, then the default group will automatically be started.
    *
    * @throws IllegalArgumentException
    *           This is thrown when data is appended out of order OR when the key contains
a invalid visibility OR when a column family is not valid for a
    *           locality group.
-   * @throws IllegalStateException
-   *           When no locality group was started.
    */
   public void append(Iterable<Entry<Key,Value>> keyValues) throws IOException
{
-    Preconditions.checkState(startedLG, "No locality group was started");
     for (Entry<Key,Value> entry : keyValues) {
       append(entry.getKey(), entry.getValue());
     }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/24465d67/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java
index 3029592..4993810 100644
--- a/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/client/rfile/RFileTest.java
@@ -104,7 +104,6 @@ public class RFileTest {
     String testFile = createTmpTestFile();
 
     try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(FileSystem.getLocal(new
Configuration())).build()) {
-      writer.startDefaultLocalityGroup();
       writer.append(testData.entrySet());
       // TODO ensure compressors are returned
     }
@@ -198,7 +197,6 @@ public class RFileTest {
     RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withTableProperties(props).build();
 
     SortedMap<Key,Value> testData1 = createTestData(10, 10, 10);
-    writer.startDefaultLocalityGroup();
     writer.append(testData1.entrySet());
     writer.close();
 
@@ -307,7 +305,6 @@ public class RFileTest {
     Value v2 = new Value("c".getBytes());
     Value v3 = new Value("t".getBytes());
 
-    writer.startDefaultLocalityGroup();
     writer.append(k1, v1);
     writer.append(k2, v2);
     writer.append(k3, v3);
@@ -345,7 +342,6 @@ public class RFileTest {
     Value v1 = new Value("p".getBytes());
     Value v2 = new Value("".getBytes());
 
-    writer.startDefaultLocalityGroup();
     writer.append(k2, v2);
     writer.append(k1, v1);
     writer.close();
@@ -409,7 +405,6 @@ public class RFileTest {
     Value v1 = new Value("p".getBytes());
     Value v2 = new Value("q".getBytes());
 
-    writer.startDefaultLocalityGroup();
     writer.append(k2, v2);
     writer.append(k1, v1);
     writer.close();
@@ -435,7 +430,6 @@ public class RFileTest {
     SamplerConfiguration sc = new SamplerConfiguration(RowSampler.class).setOptions(ImmutableMap.of("hasher",
"murmur3_32", "modulus", "19"));
 
     RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).withSampler(sc).build();
-    writer.startDefaultLocalityGroup();
     writer.append(testData1.entrySet());
     writer.close();
 
@@ -473,7 +467,6 @@ public class RFileTest {
 
     String testFile2 = createTmpTestFile();
     RFileWriter writer = RFile.newWriter().to(testFile2).build();
-    writer.startDefaultLocalityGroup();
     writer.append(scanner);
     writer.close();
     scanner.close();
@@ -517,7 +510,6 @@ public class RFileTest {
     LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
     String testFile = createTmpTestFile();
     try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build())
{
-      writer.startDefaultLocalityGroup();
       writer.append(k2, v2);
       writer.append(k1, v1);
     }
@@ -539,7 +531,6 @@ public class RFileTest {
     LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
     String testFile = createTmpTestFile();
     try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build())
{
-      writer.startDefaultLocalityGroup();
       writer.append(data);
     }
   }
@@ -570,32 +561,21 @@ public class RFileTest {
   }
 
   @Test(expected = IllegalStateException.class)
-  public void testNoLocalityGroupStarted() throws Exception {
-    LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
-    String testFile = createTmpTestFile();
-    try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build())
{
-      Key k1 = new Key("r1", "f1", "q1");
-      writer.append(k1, new Value("".getBytes()));
-    }
-  }
-
-  @Test(expected = IllegalStateException.class)
-  public void testNoLocalityGroupStartedIterable() throws Exception {
+  public void testDoubleStart() throws Exception {
     LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
     String testFile = createTmpTestFile();
     try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build())
{
-      Key k1 = new Key("r1", "f1", "q1");
-      Entry<Key,Value> entry = new AbstractMap.SimpleEntry<>(k1, new Value("".getBytes()));
-      writer.append(Collections.singletonList(entry));
+      writer.startDefaultLocalityGroup();
+      writer.startDefaultLocalityGroup();
     }
   }
 
   @Test(expected = IllegalStateException.class)
-  public void testDoubleStart() throws Exception {
+  public void testAppendStartDefault() throws Exception {
     LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
     String testFile = createTmpTestFile();
     try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build())
{
-      writer.startDefaultLocalityGroup();
+      writer.append(new Key("r1", "f1", "q1"), new Value("1".getBytes()));
       writer.startDefaultLocalityGroup();
     }
   }
@@ -605,7 +585,6 @@ public class RFileTest {
     LocalFileSystem localFs = FileSystem.getLocal(new Configuration());
     String testFile = createTmpTestFile();
     try (RFileWriter writer = RFile.newWriter().to(testFile).withFileSystem(localFs).build())
{
-      writer.startDefaultLocalityGroup();
       Key k1 = new Key("r1", "f1", "q1");
       writer.append(k1, new Value("".getBytes()));
       writer.startNewLocalityGroup("lg1", "fam1");


Mime
View raw message