incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1099891 - in /incubator/hcatalog/trunk/src: java/org/apache/hcatalog/common/HCatUtil.java test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
Date Thu, 05 May 2011 17:45:37 GMT
Author: hashutosh
Date: Thu May  5 17:45:36 2011
New Revision: 1099891

URL: http://svn.apache.org/viewvc?rev=1099891&view=rev
Log:
HCAT-18: Refactor TestRCFileInputStorageDriver

Modified:
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1099891&r1=1099890&r2=1099891&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Thu May  5
17:45:36 2011
@@ -100,6 +100,16 @@ public class HCatUtil {
     return bytes;
   }
 
+  public static List<HCatFieldSchema> getHCatFieldSchemaList(FieldSchema ... fields)
throws HCatException {
+    List<HCatFieldSchema> result = new ArrayList<HCatFieldSchema>(fields.length);
+
+    for(FieldSchema f : fields) {
+      result.add(HCatSchemaUtils.getHCatFieldSchema(f));
+    }
+
+    return result;
+  }
+
   public static List<HCatFieldSchema> getHCatFieldSchemaList(List<FieldSchema>
fields) throws HCatException {
       if(fields == null) {
           return null;

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java?rev=1099891&r1=1099890&r2=1099891&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
(original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java
Thu May  5 17:45:36 2011
@@ -15,12 +15,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hcatalog.rcfile;import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
+package org.apache.hcatalog.rcfile;
+
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.*;
 
 import junit.framework.Assert;
 import junit.framework.TestCase;
@@ -49,58 +48,54 @@ import org.apache.hcatalog.data.DefaultH
 import org.apache.hcatalog.data.HCatRecord;
 import org.apache.hcatalog.data.schema.HCatSchema;
 import org.apache.hcatalog.rcfile.RCFileInputDriver;
-;
-
-public class TestRCFileInputStorageDriver extends TestCase{
-
-  private static Configuration conf = new Configuration();
-
-  private static Path file;
 
-  private static FileSystem fs;
-
-   static {
-
-    try {
-      fs = FileSystem.getLocal(conf);
-      Path dir = new Path(System.getProperty("test.data.dir", ".") + "/mapred");
-      file = new Path(dir, "test_rcfile");
-      fs.delete(dir, true);
-    } catch (Exception e) {
-    }
-  }
 
-  public void testConvertValueToTuple() throws IOException,InterruptedException{
-    fs.delete(file, true);
+public class TestRCFileInputStorageDriver extends TestCase{
+  private static final Configuration conf = new Configuration();
+  private static final Path dir =  new Path(System.getProperty("test.data.dir", ".") + "/mapred");
+  private static final Path file = new Path(dir, "test_rcfile");
 
+  // Generate sample records to compare against
+  private byte[][][] getRecords() throws UnsupportedEncodingException {
     byte[][] record_1 = {"123".getBytes("UTF-8"), "456".getBytes("UTF-8"),
         "789".getBytes("UTF-8"), "1000".getBytes("UTF-8"),
-        "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"),
+        "5.3".getBytes("UTF-8"), "hcatalog and hadoop".getBytes("UTF-8"),
         new byte[0], "\\N".getBytes("UTF-8")};
     byte[][] record_2 = {"100".getBytes("UTF-8"), "200".getBytes("UTF-8"),
         "123".getBytes("UTF-8"), "1000".getBytes("UTF-8"),
-        "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"),
+        "5.3".getBytes("UTF-8"), "hcatalog and hadoop".getBytes("UTF-8"),
         new byte[0], "\\N".getBytes("UTF-8")};
+    return new byte[][][]{record_1, record_2};
+  }
 
+  // Write sample records to file for individual tests
+  private BytesRefArrayWritable[] initTestEnvironment() throws IOException {
+    FileSystem fs = FileSystem.getLocal(conf);
+    fs.delete(file, true);
+
+    byte [][][] records = getRecords();
     RCFileOutputFormat.setColumnNumber(conf, 8);
-    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
-        new DefaultCodec());
-    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length);
-    for (int i = 0; i < record_1.length; i++) {
-      BytesRefWritable cu = new BytesRefWritable(record_1[i], 0,
-          record_1[i].length);
+    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec());
+
+    BytesRefArrayWritable bytes = writeBytesToFile(records[0], writer);
+    BytesRefArrayWritable bytes2 = writeBytesToFile(records[1], writer);
+
+    writer.close();
+    return new BytesRefArrayWritable[]{bytes,bytes2};
+  }
+
+  private BytesRefArrayWritable writeBytesToFile(byte[][] record, RCFile.Writer writer) throws
IOException {
+    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record.length);
+    for (int i = 0; i < record.length; i++) {
+      BytesRefWritable cu = new BytesRefWritable(record[i], 0, record[i].length);
       bytes.set(i, cu);
     }
     writer.append(bytes);
-    BytesRefArrayWritable bytes2 = new BytesRefArrayWritable(record_2.length);
-    for (int i = 0; i < record_2.length; i++) {
-      BytesRefWritable cu = new BytesRefWritable(record_2[i], 0,
-          record_2[i].length);
-      bytes2.set(i, cu);
-    }
-    writer.append(bytes2);
-    writer.close();
-    BytesRefArrayWritable[] bytesArr = new BytesRefArrayWritable[]{bytes,bytes2};
+    return bytes;
+  }
+
+  public void testConvertValueToTuple() throws IOException,InterruptedException{
+    BytesRefArrayWritable[] bytesArr = initTestEnvironment();
 
     HCatSchema schema = buildHiveSchema();
     RCFileInputDriver sd = new RCFileInputDriver();
@@ -127,36 +122,7 @@ public class TestRCFileInputStorageDrive
   }
 
   public void testPruning() throws IOException,InterruptedException{
-    fs.delete(file, true);
-
-    byte[][] record_1 = {"123".getBytes("UTF-8"), "456".getBytes("UTF-8"),
-        "789".getBytes("UTF-8"), "1000".getBytes("UTF-8"),
-        "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"),
-        new byte[0], "\\N".getBytes("UTF-8")};
-    byte[][] record_2 = {"100".getBytes("UTF-8"), "200".getBytes("UTF-8"),
-        "123".getBytes("UTF-8"), "1000".getBytes("UTF-8"),
-        "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"),
-        new byte[0], "\\N".getBytes("UTF-8")};
-
-    RCFileOutputFormat.setColumnNumber(conf, 8);
-    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
-        new DefaultCodec());
-    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length);
-    for (int i = 0; i < record_1.length; i++) {
-      BytesRefWritable cu = new BytesRefWritable(record_1[i], 0,
-          record_1[i].length);
-      bytes.set(i, cu);
-    }
-    writer.append(bytes);
-    BytesRefArrayWritable bytes2 = new BytesRefArrayWritable(record_2.length);
-    for (int i = 0; i < record_2.length; i++) {
-      BytesRefWritable cu = new BytesRefWritable(record_2[i], 0,
-          record_2[i].length);
-      bytes2.set(i, cu);
-    }
-    writer.append(bytes2);
-    writer.close();
-    BytesRefArrayWritable[] bytesArr = new BytesRefArrayWritable[]{bytes,bytes2};
+    BytesRefArrayWritable[] bytesArr = initTestEnvironment();
 
     RCFileInputDriver sd = new RCFileInputDriver();
     JobContext jc = new JobContext(conf, new JobID());
@@ -185,36 +151,7 @@ public class TestRCFileInputStorageDrive
   }
 
   public void testReorderdCols() throws IOException,InterruptedException{
-    fs.delete(file, true);
-
-    byte[][] record_1 = {"123".getBytes("UTF-8"), "456".getBytes("UTF-8"),
-        "789".getBytes("UTF-8"), "1000".getBytes("UTF-8"),
-        "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"),
-        new byte[0], "\\N".getBytes("UTF-8")};
-    byte[][] record_2 = {"100".getBytes("UTF-8"), "200".getBytes("UTF-8"),
-        "123".getBytes("UTF-8"), "1000".getBytes("UTF-8"),
-        "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"),
-        new byte[0], "\\N".getBytes("UTF-8")};
-
-    RCFileOutputFormat.setColumnNumber(conf, 8);
-    RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null,
-        new DefaultCodec());
-    BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length);
-    for (int i = 0; i < record_1.length; i++) {
-      BytesRefWritable cu = new BytesRefWritable(record_1[i], 0,
-          record_1[i].length);
-      bytes.set(i, cu);
-    }
-    writer.append(bytes);
-    BytesRefArrayWritable bytes2 = new BytesRefArrayWritable(record_2.length);
-    for (int i = 0; i < record_2.length; i++) {
-      BytesRefWritable cu = new BytesRefWritable(record_2[i], 0,
-          record_2[i].length);
-      bytes2.set(i, cu);
-    }
-    writer.append(bytes2);
-    writer.close();
-    BytesRefArrayWritable[] bytesArr = new BytesRefArrayWritable[]{bytes,bytes2};
+    BytesRefArrayWritable[] bytesArr = initTestEnvironment();
 
     RCFileInputDriver sd = new RCFileInputDriver();
     JobContext jc = new JobContext(conf, new JobID());
@@ -244,122 +181,101 @@ public class TestRCFileInputStorageDrive
     }
     assertFalse(rr.nextKeyValue());
   }
-
   private HCatRecord[] getExpectedRecords(){
-
     List<Object> rec_1 = new ArrayList<Object>(8);
-    rec_1.add(new Byte("123"));
-    rec_1.add(new Short("456"));
-    rec_1.add( new Integer(789));
-    rec_1.add( new Long(1000L));
-    rec_1.add( new Double(5.3D));
-    rec_1.add( new String("howl and hadoop"));
-    rec_1.add( null);
-    rec_1.add( null);
+    Collections.addAll(rec_1, new Byte("123"),
+                              new Short("456"),
+                              new Integer(789),
+                              new Long(1000L),
+                              new Double(5.3D),
+                              new String("hcatalog and hadoop"),
+                              null,
+                              null);
 
     HCatRecord tup_1 = new DefaultHCatRecord(rec_1);
 
     List<Object> rec_2 = new ArrayList<Object>(8);
-    rec_2.add( new Byte("100"));
-    rec_2.add( new Short("200"));
-    rec_2.add( new Integer(123));
-    rec_2.add( new Long(1000L));
-    rec_2.add( new Double(5.3D));
-    rec_2.add( new String("howl and hadoop"));
-    rec_2.add( null);
-    rec_2.add( null);
+    Collections.addAll(rec_2, new Byte("100"),
+                              new Short("200"),
+                              new Integer(123),
+                              new Long(1000L),
+                              new Double(5.3D),
+                              new String("hcatalog and hadoop"),
+                              null,
+                              null);
     HCatRecord tup_2 = new DefaultHCatRecord(rec_2);
 
     return  new HCatRecord[]{tup_1,tup_2};
-
   }
 
   private HCatRecord[] getPrunedRecords(){
-
     List<Object> rec_1 = new ArrayList<Object>(8);
-    rec_1.add(new Byte("123"));
-    rec_1.add( new Integer(789));
-    rec_1.add( new Double(5.3D));
-    rec_1.add( new String("howl and hadoop"));
-    rec_1.add( null);
+    Collections.addAll(rec_1, new Byte("123"),
+        new Integer(789),
+        new Double(5.3D),
+        new String("hcatalog and hadoop"),
+        null);
     HCatRecord tup_1 = new DefaultHCatRecord(rec_1);
 
     List<Object> rec_2 = new ArrayList<Object>(8);
-    rec_2.add( new Byte("100"));
-    rec_2.add( new Integer(123));
-    rec_2.add( new Double(5.3D));
-    rec_2.add( new String("howl and hadoop"));
-    rec_2.add( null);
+    Collections.addAll(rec_2, new Byte("100"),
+        new Integer(123),
+        new Double(5.3D),
+        new String("hcatalog and hadoop"),
+        null);
     HCatRecord tup_2 = new DefaultHCatRecord(rec_2);
 
     return  new HCatRecord[]{tup_1,tup_2};
-
   }
 
   private HCatSchema buildHiveSchema() throws HCatException{
-
-    List<FieldSchema> fields = new ArrayList<FieldSchema>(8);
-    fields.add(new FieldSchema("atinyint", "tinyint", ""));
-    fields.add(new FieldSchema("asmallint", "smallint", ""));
-    fields.add(new FieldSchema("aint", "int", ""));
-    fields.add(new FieldSchema("along", "bigint", ""));
-    fields.add(new FieldSchema("adouble", "double", ""));
-    fields.add(new FieldSchema("astring", "string", ""));
-    fields.add(new FieldSchema("anullint", "int", ""));
-    fields.add(new FieldSchema("anullstring", "string", ""));
-
-    return new HCatSchema(HCatUtil.getHCatFieldSchemaList(fields));
+    return new HCatSchema(HCatUtil.getHCatFieldSchemaList(new FieldSchema("atinyint", "tinyint",
""),
+                                                          new FieldSchema("asmallint", "smallint",
""),
+                                                          new FieldSchema("aint", "int",
""),
+                                                          new FieldSchema("along", "bigint",
""),
+                                                          new FieldSchema("adouble", "double",
""),
+                                                          new FieldSchema("astring", "string",
""),
+                                                          new FieldSchema("anullint", "int",
""),
+                                                          new FieldSchema("anullstring",
"string", "")));
   }
 
   private HCatSchema buildPrunedSchema() throws HCatException{
-
-    List<FieldSchema> fields = new ArrayList<FieldSchema>(5);
-    fields.add(new FieldSchema("atinyint", "tinyint", ""));
-    fields.add(new FieldSchema("aint", "int", ""));
-    fields.add(new FieldSchema("adouble", "double", ""));
-    fields.add(new FieldSchema("astring", "string", ""));
-    fields.add(new FieldSchema("anullint", "int", ""));
-
-    return new HCatSchema(HCatUtil.getHCatFieldSchemaList(fields));
+    return new HCatSchema(HCatUtil.getHCatFieldSchemaList(new FieldSchema("atinyint", "tinyint",
""),
+                                                          new FieldSchema("aint", "int",
""),
+                                                          new FieldSchema("adouble", "double",
""),
+                                                          new FieldSchema("astring", "string",
""),
+                                                          new FieldSchema("anullint", "int",
"")));
   }
 
   private HCatSchema buildReorderedSchema() throws HCatException{
-
-    List<FieldSchema> fields = new ArrayList<FieldSchema>(7);
-
-    fields.add(new FieldSchema("aint", "int", ""));
-    fields.add(new FieldSchema("part1", "string", ""));
-    fields.add(new FieldSchema("adouble", "double", ""));
-    fields.add(new FieldSchema("newCol", "tinyint", ""));
-    fields.add(new FieldSchema("astring", "string", ""));
-    fields.add(new FieldSchema("atinyint", "tinyint", ""));
-    fields.add(new FieldSchema("anullint", "int", ""));
-
-
-    return new HCatSchema(HCatUtil.getHCatFieldSchemaList(fields));
+    return new HCatSchema(HCatUtil.getHCatFieldSchemaList(new FieldSchema("aint", "int",
""),
+                                                          new FieldSchema("part1", "string",
""),
+                                                          new FieldSchema("adouble", "double",
""),
+                                                          new FieldSchema("newCol", "tinyint",
""),
+                                                          new FieldSchema("astring", "string",
""),
+                                                          new FieldSchema("atinyint", "tinyint",
""),
+                                                          new FieldSchema("anullint", "int",
"")));
   }
 
   private HCatRecord[] getReorderedCols(){
-
     List<Object> rec_1 = new ArrayList<Object>(7);
-    rec_1.add( new Integer(789));
-    rec_1.add( new String("first-part"));
-    rec_1.add( new Double(5.3D));
-    rec_1.add( null); // new column
-    rec_1.add( new String("howl and hadoop"));
-    rec_1.add( new Byte("123"));
-    rec_1.add( null);
+    Collections.addAll(rec_1,  new Integer(789),
+                               new String("first-part"),
+                               new Double(5.3D),
+                               null, // new column
+                               new String("hcatalog and hadoop"),
+                               new Byte("123"),
+                               null);
     HCatRecord tup_1 = new DefaultHCatRecord(rec_1);
 
     List<Object> rec_2 = new ArrayList<Object>(7);
-
-    rec_2.add( new Integer(123));
-    rec_2.add( new String("first-part"));
-    rec_2.add( new Double(5.3D));
-    rec_2.add(null);
-    rec_2.add( new String("howl and hadoop"));
-    rec_2.add( new Byte("100"));
-    rec_2.add( null);
+    Collections.addAll(rec_2, new Integer(123),
+        new String("first-part"),
+        new Double(5.3D),
+        null,
+        new String("hcatalog and hadoop"),
+        new Byte("100"),
+        null);
     HCatRecord tup_2 = new DefaultHCatRecord(rec_2);
 
     return  new HCatRecord[]{tup_1,tup_2};



Mime
View raw message