accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ctubb...@apache.org
Subject [01/21] accumulo git commit: ACCUMULO-4119 Use DataFileValue consistently
Date Wed, 03 Feb 2016 03:08:59 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.6 a95127d1d -> 46958bf7b
  refs/heads/1.7 6cc5ab56b -> a706b019a
  refs/heads/master b4cd908a3 -> 2c8088e99


ACCUMULO-4119 Use DataFileValue consistently

* Ensure DataFileValue encodes/decodes with UTF-8
* Ensure usage of DataFileValue instead of manual parsing
* Provide more encoding utility methods (String, Value)


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c462001a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c462001a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c462001a

Branch: refs/heads/1.6
Commit: c462001a71cf4e4e5b9b0865cb6d0036bf4d84c7
Parents: a95127d
Author: Christopher Tubbs <ctubbsii@apache.org>
Authored: Wed Jan 27 19:08:56 2016 -0500
Committer: Christopher Tubbs <ctubbsii@apache.org>
Committed: Tue Feb 2 18:24:28 2016 -0500

----------------------------------------------------------------------
 .../core/metadata/schema/DataFileValue.java     | 18 ++++++++++++---
 .../org/apache/accumulo/core/util/Merge.java    |  8 ++-----
 .../apache/accumulo/server/util/CloneTest.java  | 24 ++++++++++++++------
 .../metadata/MetadataBatchScanTest.java         |  3 ++-
 4 files changed, 36 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/c462001a/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java
index cebe041..dfa1114 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java
@@ -16,6 +16,10 @@
  */
 package org.apache.accumulo.core.metadata.schema;
 
+import static com.google.common.base.Charsets.UTF_8;
+
+import org.apache.accumulo.core.data.Value;
+
 public class DataFileValue {
   private long size;
   private long numEntries;
@@ -34,7 +38,7 @@ public class DataFileValue {
   }
 
   public DataFileValue(byte[] encodedDFV) {
-    String[] ba = new String(encodedDFV).split(",");
+    String[] ba = new String(encodedDFV, UTF_8).split(",");
 
     size = Long.parseLong(ba[0]);
     numEntries = Long.parseLong(ba[1]);
@@ -62,9 +66,17 @@ public class DataFileValue {
   }
 
   public byte[] encode() {
+    return encodeAsString().getBytes(UTF_8);
+  }
+
+  public String encodeAsString() {
     if (time >= 0)
-      return ("" + size + "," + numEntries + "," + time).getBytes();
-    return ("" + size + "," + numEntries).getBytes();
+      return ("" + size + "," + numEntries + "," + time);
+    return ("" + size + "," + numEntries);
+  }
+
+  public Value encodeAsValue() {
+    return new Value(encode());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c462001a/core/src/main/java/org/apache/accumulo/core/util/Merge.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/Merge.java b/core/src/main/java/org/apache/accumulo/core/util/Merge.java
index c3bc678..c5e3e8e 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/Merge.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/Merge.java
@@ -16,8 +16,6 @@
  */
 package org.apache.accumulo.core.util;
 
-import static com.google.common.base.Charsets.UTF_8;
-
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -34,6 +32,7 @@ import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 import org.apache.accumulo.core.security.Authorizations;
@@ -238,10 +237,7 @@ public class Merge {
           Entry<Key,Value> entry = iterator.next();
           Key key = entry.getKey();
           if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
-            String[] sizeEntries = new String(entry.getValue().get(), UTF_8).split(",");
-            if (sizeEntries.length == 2) {
-              tabletSize += Long.parseLong(sizeEntries[0]);
-            }
+            tabletSize += new DataFileValue(entry.getValue().get()).getSize();
           } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
             KeyExtent extent = new KeyExtent(key.getRow(), entry.getValue());
             return new Size(extent, tabletSize);

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c462001a/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java
index ebb83bf..74d5e64 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/util/CloneTest.java
@@ -16,11 +16,12 @@
  */
 package org.apache.accumulo.server.util;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import java.util.HashSet;
 import java.util.Map.Entry;
 
-import junit.framework.TestCase;
-
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.BatchWriterConfig;
 import org.apache.accumulo.core.client.Connector;
@@ -32,13 +33,16 @@ import org.apache.accumulo.core.data.KeyExtent;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.hadoop.io.Text;
+import org.junit.Test;
 
-public class CloneTest extends TestCase {
+public class CloneTest {
 
+  @Test
   public void testNoFiles() throws Exception {
     MockInstance mi = new MockInstance();
     Connector conn = mi.getConnector("", new PasswordToken(""));
@@ -67,6 +71,7 @@ public class CloneTest extends TestCase {
 
   }
 
+  @Test
   public void testFilesChange() throws Exception {
     MockInstance mi = new MockInstance();
     Connector conn = mi.getConnector("", new PasswordToken(""));
@@ -76,7 +81,7 @@ public class CloneTest extends TestCase {
 
     TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0".getBytes()));
     TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value("/default_tablet".getBytes()));
-    mut.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf", "1,200");
+    mut.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf", new DataFileValue(1,
200).encodeAsString());
 
     BatchWriter bw1 = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
 
@@ -90,7 +95,7 @@ public class CloneTest extends TestCase {
 
     Mutation mut2 = new Mutation(ke.getMetadataEntry());
     mut2.putDelete(DataFileColumnFamily.NAME.toString(), "/default_tablet/0_0.rf");
-    mut2.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/1_0.rf", "2,300");
+    mut2.put(DataFileColumnFamily.NAME.toString(), "/default_tablet/1_0.rf", new DataFileValue(2,
300).encodeAsString());
 
     bw1.addMutation(mut2);
     bw1.flush();
@@ -119,6 +124,7 @@ public class CloneTest extends TestCase {
   }
 
   // test split where files of children are the same
+  @Test
   public void testSplit1() throws Exception {
     MockInstance mi = new MockInstance();
     Connector conn = mi.getConnector("", new PasswordToken(""));
@@ -161,6 +167,7 @@ public class CloneTest extends TestCase {
   }
 
   // test split where files of children differ... like majc and split occurred
+  @Test
   public void testSplit2() throws Exception {
     MockInstance mi = new MockInstance();
     Connector conn = mi.getConnector("", new PasswordToken(""));
@@ -226,12 +233,13 @@ public class CloneTest extends TestCase {
 
     TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mut, new Value("M0".getBytes()));
     TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes()));
-    mut.put(DataFileColumnFamily.NAME.toString(), file, "10,200");
+    mut.put(DataFileColumnFamily.NAME.toString(), file, new DataFileValue(10, 200).encodeAsString());
 
     return mut;
   }
 
   // test two tablets splitting into four
+  @Test
   public void testSplit3() throws Exception {
     MockInstance mi = new MockInstance();
     Connector conn = mi.getConnector("", new PasswordToken(""));
@@ -278,6 +286,7 @@ public class CloneTest extends TestCase {
   }
 
   // test cloned marker
+  @Test
   public void testClonedMarker() throws Exception {
 
     MockInstance mi = new MockInstance();
@@ -343,6 +352,7 @@ public class CloneTest extends TestCase {
   }
 
   // test two tablets splitting into four
+  @Test
   public void testMerge() throws Exception {
     MockInstance mi = new MockInstance();
     Connector conn = mi.getConnector("", new PasswordToken(""));
@@ -360,7 +370,7 @@ public class CloneTest extends TestCase {
 
     bw1.addMutation(deleteTablet("0", "m", null, "/d1", "/d1/file1"));
     Mutation mut = createTablet("0", null, null, "/d2", "/d2/file2");
-    mut.put(DataFileColumnFamily.NAME.toString(), "/d1/file1", "10,200");
+    mut.put(DataFileColumnFamily.NAME.toString(), "/d1/file1", new DataFileValue(10, 200).encodeAsString());
     bw1.addMutation(mut);
 
     bw1.flush();

http://git-wip-us.apache.org/repos/asf/accumulo/blob/c462001a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
index d764d35..07bfdc7 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/metadata/MetadataBatchScanTest.java
@@ -39,6 +39,7 @@ import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
 import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
 import org.apache.accumulo.core.security.Authorizations;
@@ -108,7 +109,7 @@ public class MetadataBatchScanTest {
         TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(mut, new Value(dir.getBytes(UTF_8)));
 
         for (int i = 0; i < 5; i++) {
-          mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"),
new Value("10000,1000000".getBytes(UTF_8)));
+          mut.put(DataFileColumnFamily.NAME, new Text(dir + "/00000_0000" + i + ".map"),
new DataFileValue(10000, 1000000).encodeAsValue());
         }
 
         bw.addMutation(mut);


Mime
View raw message