asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject [7/8] asterixdb git commit: RAT Cleanup, Resulting LICENSE fixes
Date Thu, 02 Feb 2017 16:54:01 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js
deleted file mode 100644
index d8b79df..0000000
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js
+++ /dev/null
@@ -1 +0,0 @@
-(function(B){var A={series:{threshold:null}};function C(D){function E(L,S,M){if(!S.threshold){return }var F=M.pointsize,I,O,N,G,K,H=B.extend({},S);H.datapoints={points:[],pointsize:F};H.label=null;H.color=S.threshold.color;H.threshold=null;H.originSeries=S;H.data=[];var P=S.threshold.below,Q=M.points,R=S.lines.show;threspoints=[];newpoints=[];for(I=0;I<Q.length;I+=F){O=Q[I];N=Q[I+1];K=G;if(N<P){G=threspoints}else{G=newpoints}if(R&&K!=G&&O!=null&&I>0&&Q[I-F]!=null){var J=(O-Q[I-F])/(N-Q[I-F+1])*(P-N)+O;K.push(J);K.push(P);for(m=2;m<F;++m){K.push(Q[I+m])}G.push(null);G.push(null);for(m=2;m<F;++m){G.push(Q[I+m])}G.push(J);G.push(P);for(m=2;m<F;++m){G.push(Q[I+m])}}G.push(O);G.push(N)}M.points=newpoints;H.datapoints.points=threspoints;if(H.datapoints.points.length>0){L.getData().push(H)}}D.hooks.processDatapoints.push(E)}B.plot.plugins.push({init:C,options:A,name:"threshold",version:"1.0"})})(jQuery);
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
index 8caa9ec..6cfaaa5 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
@@ -41,6 +41,20 @@
     <root.dir>${basedir}/../..</root.dir>
   </properties>
 
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <exclude>src/test/resources/data/beer.txt</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
   <dependencies>
     <dependency>
       <groupId>org.apache.hyracks</groupId>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
index 73086cd..b88d292 100644
--- a/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-dist/pom.xml
@@ -50,7 +50,7 @@
         <executions>
           <execution>
             <id>copy-scripts</id>
-            <phase>package</phase>
+            <phase>generate-test-resources</phase>
             <goals>
               <goal>copy-resources</goal>
             </goals>
@@ -61,7 +61,6 @@
                   <directory>src/main/resources</directory>
                 </resource>
               </resources>
-              <directoryMode>0755</directoryMode>
             </configuration>
           </execution>
         </executions>
@@ -73,7 +72,7 @@
         <executions>
           <execution>
             <id>process-test-classes</id>
-            <phase>package</phase>
+            <phase>process-test-resources</phase>
             <configuration>
               <target>
                 <chmod file="target/appassembler/bin/*" perm="755" />
@@ -85,6 +84,16 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <exclude>src/main/resources/conf/master</exclude>
+            <exclude>src/main/resources/conf/slaves</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 </project>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
index d1f1c70..21e22ab 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/hyracks-integration-tests/pom.xml
@@ -37,6 +37,15 @@
           <skip>true</skip>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <exclude>data/**</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
   <properties>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
index 1327912..394fa11 100644
--- a/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-examples/text-example/textserver/pom.xml
@@ -99,6 +99,16 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <exclude>data/file1.txt</exclude>
+            <exclude>data/file2.txt</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
   <profiles>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
index 9c25d13..b4abc4a 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/pom.xml
@@ -33,25 +33,6 @@
   <build>
     <plugins>
       <plugin>
-        <artifactId>maven-clean-plugin</artifactId>
-        <version>2.5</version>
-        <configuration>
-          <filesets>
-            <fileset>
-              <directory>.</directory>
-              <includes>
-                <include>edu*</include>
-                <include>actual*</include>
-                <include>build*</include>
-                <include>expect*</include>
-                <include>ClusterController*</include>
-                <include>edu.uci.*</include>
-              </includes>
-            </fileset>
-          </filesets>
-        </configuration>
-      </plugin>
-      <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
         <version>2.2</version>
@@ -81,6 +62,16 @@
           </execution>
         </executions>
       </plugin>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <exclude>src/test/resources/data/customer.tbl</exclude>
+            <exclude>src/test/resources/expected/part-0</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 
@@ -409,5 +400,11 @@
       <artifactId>commons-io</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hyracks</groupId>
+      <artifactId>hyracks-test-support</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java
index c5ca25e..2307a43 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/dataflow/DataflowTest.java
@@ -19,6 +19,8 @@
 
 package org.apache.hyracks.hdfs.dataflow;
 
+import static org.apache.hyracks.test.support.TestUtils.joinPath;
+
 import java.io.DataOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -67,15 +69,19 @@ import org.apache.hyracks.hdfs.utils.TestUtils;
 @SuppressWarnings({ "deprecation" })
 public class DataflowTest extends TestCase {
 
-    private static final String ACTUAL_RESULT_DIR = "actual";
-    private static final String EXPECTED_RESULT_PATH = "src/test/resources/expected";
-    private static final String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf";
+    protected static final String ACTUAL_RESULT_DIR = joinPath("target", "actual");
+    private static final String TEST_RESOURCES = joinPath("src", "test", "resources");
+    protected static final String EXPECTED_RESULT_PATH = joinPath(TEST_RESOURCES, "expected");
+    private static final String PATH_TO_HADOOP_CONF = joinPath(TEST_RESOURCES, "hadoop", "conf");
+    protected static final String BUILD_DIR = joinPath("target", "build");
+
 
-    private static final String DATA_PATH = "src/test/resources/data/customer.tbl";
-    private static final String HDFS_INPUT_PATH = "/customer/";
-    private static final String HDFS_OUTPUT_PATH = "/customer_result/";
+    private static final String DATA_PATH = joinPath(TEST_RESOURCES, "data", "customer.tbl");
+    protected static final String HDFS_INPUT_PATH = "/customer/";
+    protected static final String HDFS_OUTPUT_PATH = "/customer_result/";
 
     private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml";
+    private static final String MINIDFS_BASEDIR = joinPath("target", "hdfs");
     private MiniDFSCluster dfsCluster;
 
     private JobConf conf = new JobConf();
@@ -91,10 +97,16 @@ public class DataflowTest extends TestCase {
     }
 
     private void cleanupStores() throws IOException {
-        FileUtils.forceMkdir(new File("teststore"));
-        FileUtils.forceMkdir(new File("build"));
-        FileUtils.cleanDirectory(new File("teststore"));
-        FileUtils.cleanDirectory(new File("build"));
+        FileUtils.forceMkdir(new File(MINIDFS_BASEDIR));
+        FileUtils.cleanDirectory(new File(MINIDFS_BASEDIR));
+    }
+
+    protected Configuration getConfiguration() {
+        return conf;
+    }
+
+    protected MiniDFSCluster getMiniDFSCluster(Configuration conf, int numberOfNC) throws IOException {
+        return new MiniDFSCluster(conf, numberOfNC, true, null);
     }
 
     /**
@@ -102,16 +114,17 @@ public class DataflowTest extends TestCase {
      *
      * @throws IOException
      */
-    private void startHDFS() throws IOException {
-        conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
-        conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
-        conf.addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
+    protected void startHDFS() throws IOException {
+        getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
+        getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
+        getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
 
         FileSystem lfs = FileSystem.getLocal(new Configuration());
-        lfs.delete(new Path("build"), true);
-        System.setProperty("hadoop.log.dir", "logs");
-        dfsCluster = new MiniDFSCluster(conf, numberOfNC, true, null);
-        FileSystem dfs = FileSystem.get(conf);
+        lfs.delete(new Path(BUILD_DIR), true);
+        System.setProperty("hadoop.log.dir", joinPath("target", "logs"));
+        getConfiguration().set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, MINIDFS_BASEDIR);
+        dfsCluster = getMiniDFSCluster(getConfiguration(), numberOfNC);
+        FileSystem dfs = FileSystem.get(getConfiguration());
         Path src = new Path(DATA_PATH);
         Path dest = new Path(HDFS_INPUT_PATH);
         Path result = new Path(HDFS_OUTPUT_PATH);
@@ -120,7 +133,7 @@ public class DataflowTest extends TestCase {
         dfs.copyFromLocalFile(src, dest);
 
         DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
-        conf.writeXml(confOutput);
+        getConfiguration().writeXml(confOutput);
         confOutput.flush();
         confOutput.close();
     }
@@ -178,14 +191,14 @@ public class DataflowTest extends TestCase {
      * @return true if correct
      * @throws Exception
      */
-    private boolean checkResults() throws Exception {
-        FileSystem dfs = FileSystem.get(conf);
+    protected boolean checkResults() throws Exception {
+        FileSystem dfs = FileSystem.get(getConfiguration());
         Path result = new Path(HDFS_OUTPUT_PATH);
         Path actual = new Path(ACTUAL_RESULT_DIR);
         dfs.copyToLocalFile(result, actual);
 
-        TestUtils.compareWithResult(new File(EXPECTED_RESULT_PATH + File.separator + "part-0"), new File(
-                ACTUAL_RESULT_DIR + File.separator + "customer_result" + File.separator + "part-0"));
+        TestUtils.compareWithResult(new File(joinPath(EXPECTED_RESULT_PATH, "part-0")), new File(
+                joinPath(ACTUAL_RESULT_DIR, "customer_result", "part-0")));
         return true;
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java
index 09e3a35..d522b2b 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs/utils/TestUtils.java
@@ -29,7 +29,7 @@ import org.apache.hyracks.api.client.NodeControllerInfo;
 import org.apache.hyracks.api.client.NodeStatus;
 import org.apache.hyracks.api.comm.NetworkAddress;
 
-public class TestUtils {
+public class TestUtils extends org.apache.hyracks.test.support.TestUtils {
 
     public static void compareWithResult(File expectedFile, File actualFile) throws Exception {
         BufferedReader readerExpected = new BufferedReader(new FileReader(expectedFile));

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java
index fb6f32e..3c9b1c0 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/java/org/apache/hyracks/hdfs2/dataflow/DataflowTest.java
@@ -19,18 +19,10 @@
 
 package org.apache.hyracks.hdfs2.dataflow;
 
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
 import java.util.List;
 
 import junit.framework.Assert;
-import junit.framework.TestCase;
-
-import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.mapreduce.InputFormat;
@@ -40,7 +32,6 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.ReflectionUtils;
-
 import org.apache.hyracks.api.client.HyracksConnection;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.constraints.PartitionConstraintHelper;
@@ -48,6 +39,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
 import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
 import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.job.JobId;
 import org.apache.hyracks.api.job.JobSpecification;
 import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
@@ -61,73 +53,32 @@ import org.apache.hyracks.hdfs.lib.RawBinaryHashFunctionFactory;
 import org.apache.hyracks.hdfs.lib.TextKeyValueParserFactory;
 import org.apache.hyracks.hdfs.lib.TextTupleWriterFactory;
 import org.apache.hyracks.hdfs.utils.HyracksUtils;
-import org.apache.hyracks.hdfs.utils.TestUtils;
 import org.apache.hyracks.hdfs2.scheduler.Scheduler;
 
 /**
  * Test the org.apache.hyracks.hdfs2.dataflow package,
  * the operators for the Hadoop new API.
  */
-public class DataflowTest extends TestCase {
-
-    private static final String ACTUAL_RESULT_DIR = "actual";
-    private static final String EXPECTED_RESULT_PATH = "src/test/resources/expected";
-    private static final String PATH_TO_HADOOP_CONF = "src/test/resources/hadoop/conf";
+public class DataflowTest extends org.apache.hyracks.hdfs.dataflow.DataflowTest {
 
-    private static final String DATA_PATH = "src/test/resources/data/customer.tbl";
-    private static final String HDFS_INPUT_PATH = "/customer/";
-    private static final String HDFS_OUTPUT_PATH = "/customer_result/";
-
-    private static final String HADOOP_CONF_PATH = ACTUAL_RESULT_DIR + File.separator + "conf.xml";
-    private MiniDFSCluster dfsCluster;
     private MiniDFSClusterFactory dfsClusterFactory = new MiniDFSClusterFactory();
 
     private Job conf;
-    private int numberOfNC = 2;
 
     @Override
     public void setUp() throws Exception {
         conf = new Job();
-        cleanupStores();
-        HyracksUtils.init();
-        FileUtils.forceMkdir(new File(ACTUAL_RESULT_DIR));
-        FileUtils.cleanDirectory(new File(ACTUAL_RESULT_DIR));
-        startHDFS();
+        super.setUp();
     }
 
-    private void cleanupStores() throws IOException {
-        FileUtils.forceMkdir(new File("teststore"));
-        FileUtils.forceMkdir(new File("build"));
-        FileUtils.cleanDirectory(new File("teststore"));
-        FileUtils.cleanDirectory(new File("build"));
+    @Override
+    protected Configuration getConfiguration() {
+        return conf.getConfiguration();
     }
 
-    /**
-     * Start the HDFS cluster and setup the data files
-     *
-     * @throws IOException
-     */
-    private void startHDFS() throws IOException {
-        conf.getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/core-site.xml"));
-        conf.getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/mapred-site.xml"));
-        conf.getConfiguration().addResource(new Path(PATH_TO_HADOOP_CONF + "/hdfs-site.xml"));
-
-        FileSystem lfs = FileSystem.getLocal(new Configuration());
-        lfs.delete(new Path("build"), true);
-        System.setProperty("hadoop.log.dir", "logs");
-        dfsCluster = dfsClusterFactory.getMiniDFSCluster(conf.getConfiguration(), numberOfNC);
-        FileSystem dfs = FileSystem.get(conf.getConfiguration());
-        Path src = new Path(DATA_PATH);
-        Path dest = new Path(HDFS_INPUT_PATH);
-        Path result = new Path(HDFS_OUTPUT_PATH);
-        dfs.mkdirs(dest);
-        dfs.mkdirs(result);
-        dfs.copyFromLocalFile(src, dest);
-
-        DataOutputStream confOutput = new DataOutputStream(new FileOutputStream(new File(HADOOP_CONF_PATH)));
-        conf.getConfiguration().writeXml(confOutput);
-        confOutput.flush();
-        confOutput.close();
+    @Override
+    protected MiniDFSCluster getMiniDFSCluster(Configuration conf, int numberOfNC) throws HyracksDataException {
+        return dfsClusterFactory.getMiniDFSCluster(conf, numberOfNC);
     }
 
     /**
@@ -142,7 +93,7 @@ public class DataflowTest extends TestCase {
         conf.setInputFormatClass(TextInputFormat.class);
 
         Scheduler scheduler = new Scheduler(HyracksUtils.CC_HOST, HyracksUtils.TEST_HYRACKS_CC_CLIENT_PORT);
-        InputFormat inputFormat = ReflectionUtils.newInstance(conf.getInputFormatClass(), conf.getConfiguration());
+        InputFormat inputFormat = ReflectionUtils.newInstance(conf.getInputFormatClass(), getConfiguration());
         List<InputSplit> splits = inputFormat.getSplits(conf);
 
         String[] readSchedule = scheduler.getLocationConstraints(splits);
@@ -178,35 +129,4 @@ public class DataflowTest extends TestCase {
 
         Assert.assertEquals(true, checkResults());
     }
-
-    /**
-     * Check if the results are correct
-     *
-     * @return true if correct
-     * @throws Exception
-     */
-    private boolean checkResults() throws Exception {
-        FileSystem dfs = FileSystem.get(conf.getConfiguration());
-        Path result = new Path(HDFS_OUTPUT_PATH);
-        Path actual = new Path(ACTUAL_RESULT_DIR);
-        dfs.copyToLocalFile(result, actual);
-
-        TestUtils.compareWithResult(new File(EXPECTED_RESULT_PATH + File.separator + "part-0"), new File(
-                ACTUAL_RESULT_DIR + File.separator + "customer_result" + File.separator + "part-0"));
-        return true;
-    }
-
-    /**
-     * cleanup hdfs cluster
-     */
-    private void cleanupHDFS() throws Exception {
-        dfsCluster.shutdown();
-    }
-
-    @Override
-    public void tearDown() throws Exception {
-        HyracksUtils.deinit();
-        cleanupHDFS();
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml
index 1628544..be43c5d 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/core-site.xml
@@ -1,4 +1,5 @@
 <?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
  ! Licensed to the Apache Software Foundation (ASF) under one
  ! or more contributor license agreements.  See the NOTICE file
@@ -18,8 +19,6 @@
  ! under the License.
  !-->
 
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
 <!-- Put site-specific property overrides in this file. -->
 
 <configuration>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml
index 8784730..9dca49a 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/hdfs-site.xml
@@ -1,4 +1,5 @@
 <?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
  ! Licensed to the Apache Software Foundation (ASF) under one
  ! or more contributor license agreements.  See the NOTICE file
@@ -18,8 +19,6 @@
  ! under the License.
  !-->
 
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
 <!-- Put site-specific property overrides in this file. -->
 
 <configuration>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml
index 3e8bfe4..e765c65 100644
--- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml
+++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/test/resources/hadoop/conf/mapred-site.xml
@@ -1,4 +1,5 @@
 <?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
  ! Licensed to the Apache Software Foundation (ASF) under one
  ! or more contributor license agreements.  See the NOTICE file
@@ -18,8 +19,6 @@
  ! under the License.
  !-->
 
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
 <!-- Put site-specific property overrides in this file. -->
 
 <configuration>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java
deleted file mode 100644
index e031a51..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenThread.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.IOException;
-import java.util.Random;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-
-/**
- * Quick & dirty data generator for multi-thread testing.
- */
-@SuppressWarnings("rawtypes")
-public class DataGenThread extends Thread {
-    public final BlockingQueue<TupleBatch> tupleBatchQueue;
-    private final int maxNumBatches;
-    private final int maxOutstandingBatches;
-    private int numBatches = 0;
-    private final Random rnd;
-
-    // maxOutstandingBatches pre-created tuple-batches for populating the queue.
-    private TupleBatch[] tupleBatches;
-    private int ringPos;
-
-    public DataGenThread(int numConsumers, int maxNumBatches, int batchSize, ISerializerDeserializer[] fieldSerdes,
-            int payloadSize, int rndSeed, int maxOutstandingBatches, boolean sorted) {
-        this.maxNumBatches = maxNumBatches;
-        this.maxOutstandingBatches = maxOutstandingBatches;
-        rnd = new Random(rndSeed);
-        tupleBatches = new TupleBatch[maxOutstandingBatches];
-        IFieldValueGenerator[] fieldGens = DataGenUtils.getFieldGensFromSerdes(fieldSerdes, rnd, sorted);
-        for (int i = 0; i < maxOutstandingBatches; i++) {
-            tupleBatches[i] = new TupleBatch(batchSize, fieldGens, fieldSerdes, payloadSize);
-        }
-        tupleBatchQueue = new LinkedBlockingQueue<TupleBatch>(maxOutstandingBatches);
-        ringPos = 0;
-    }
-
-    public DataGenThread(int numConsumers, int maxNumBatches, int batchSize, ISerializerDeserializer[] fieldSerdes,
-            IFieldValueGenerator[] fieldGens, int rndSeed, int maxOutstandingBatches) {
-        this.maxNumBatches = maxNumBatches;
-        this.maxOutstandingBatches = maxOutstandingBatches;
-        rnd = new Random(rndSeed);
-        tupleBatches = new TupleBatch[maxOutstandingBatches];
-        for (int i = 0; i < maxOutstandingBatches; i++) {
-            tupleBatches[i] = new TupleBatch(batchSize, fieldGens, fieldSerdes, 0);
-        }
-        tupleBatchQueue = new LinkedBlockingQueue<TupleBatch>(maxOutstandingBatches);
-        ringPos = 0;
-    }
-
-    @Override
-    public void run() {
-        while (numBatches < maxNumBatches) {
-            boolean added = false;
-            try {
-                if (tupleBatches[ringPos].inUse.compareAndSet(false, true)) {
-                    tupleBatches[ringPos].generate();
-                    tupleBatchQueue.put(tupleBatches[ringPos]);
-                    added = true;
-                }
-            } catch (IOException e) {
-                e.printStackTrace();
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-            if (added) {
-                numBatches++;
-                ringPos++;
-                if (ringPos >= maxOutstandingBatches) {
-                    ringPos = 0;
-                }
-            }
-        }
-    }
-
-    public TupleBatch getBatch() throws InterruptedException {
-        return tupleBatchQueue.take();
-    }
-
-    public void releaseBatch(TupleBatch batch) {
-        batch.inUse.set(false);
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java
deleted file mode 100644
index 130f7e2..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DataGenUtils.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-
-@SuppressWarnings("rawtypes")
-public class DataGenUtils {
-    public static IFieldValueGenerator getFieldGenFromSerde(ISerializerDeserializer serde, Random rnd, boolean sorted) {
-        if (serde instanceof IntegerSerializerDeserializer) {
-            if (sorted) {
-                return new SortedIntegerFieldValueGenerator();
-            } else {
-                return new IntegerFieldValueGenerator(rnd);
-            }
-        } else if (serde instanceof FloatSerializerDeserializer) {
-            if (sorted) {
-                return new SortedFloatFieldValueGenerator();
-            } else {
-                return new FloatFieldValueGenerator(rnd);
-            }
-        } else if (serde instanceof DoubleSerializerDeserializer) {
-            if (sorted) {
-                return new SortedDoubleFieldValueGenerator();
-            } else {
-                return new DoubleFieldValueGenerator(rnd);
-            }
-        } else if (serde instanceof UTF8StringSerializerDeserializer) {
-            return new StringFieldValueGenerator(20, rnd);
-        }
-        return null;
-    }
-
-    public static IFieldValueGenerator[] getFieldGensFromSerdes(ISerializerDeserializer[] serdes, Random rnd,
-            boolean sorted) {
-        IFieldValueGenerator[] fieldValueGens = new IFieldValueGenerator[serdes.length];
-        for (int i = 0; i < serdes.length; i++) {
-            fieldValueGens[i] = getFieldGenFromSerde(serdes[i], rnd, sorted);
-        }
-        return fieldValueGens;
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java
deleted file mode 100644
index 1e8077c..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DocumentStringFieldValueGenerator.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hyracks.util.MathUtil;
-
-public class DocumentStringFieldValueGenerator implements IFieldValueGenerator<String> {
-    private final String FIRST_NAMES_FILE = "dist.all.first.cleaned";
-    private final String LAST_NAMES_FILE = "dist.all.last.cleaned";
-
-    private final int docMinWords;
-    private final int docMaxWords;
-    private final int maxDictionarySize;
-    private final Random rnd;
-    private int[] cumulIntRanges;
-
-    private List<String> tokenDict = new ArrayList<String>();
-
-    public DocumentStringFieldValueGenerator(int docMinWords, int docMaxWords, int maxDictionarySize, Random rnd)
-            throws IOException {
-        this.docMinWords = docMinWords;
-        this.docMaxWords = docMaxWords;
-        this.maxDictionarySize = maxDictionarySize;
-        this.rnd = rnd;
-        initDictionary();
-        double[] zipfProbDist = ProbabilityHelper.getZipfProbDist(tokenDict.size(), 1);
-        cumulIntRanges = ProbabilityHelper.getCumulIntRanges(zipfProbDist);
-    }
-
-    private void initDictionary() throws IOException {
-        String line;
-        int count = 0;
-
-        // Read first names from data file.
-        InputStream firstNamesIn = this.getClass().getClassLoader().getResourceAsStream(FIRST_NAMES_FILE);
-        BufferedReader firstNamesReader = new BufferedReader(new InputStreamReader(firstNamesIn));
-        try {
-            while (count < maxDictionarySize && (line = firstNamesReader.readLine()) != null) {
-                tokenDict.add(line.trim());
-                count++;
-            }
-        } finally {
-            firstNamesReader.close();
-        }
-
-        // Read last names from data file.
-        InputStream lastNamesIn = this.getClass().getClassLoader().getResourceAsStream(LAST_NAMES_FILE);
-        BufferedReader lastNamesReader = new BufferedReader(new InputStreamReader(lastNamesIn));
-        try {
-            while (count < maxDictionarySize && (line = lastNamesReader.readLine()) != null) {
-                tokenDict.add(line.trim());
-                count++;
-            }
-        } finally {
-            lastNamesReader.close();
-        }
-    }
-
-    @Override
-    public String next() {
-        StringBuilder strBuilder = new StringBuilder();
-        int numWords = MathUtil.stripSignBit(rnd.nextInt()) % (docMaxWords - docMinWords + 1) + docMinWords;
-        for (int i = 0; i < numWords; i++) {
-            int ix = ProbabilityHelper.choose(cumulIntRanges, rnd.nextInt());
-            strBuilder.append(tokenDict.get(ix));
-            if (i != numWords - 1) {
-                strBuilder.append(" ");
-            }
-        }
-        return strBuilder.toString();
-    }
-
-    public List<String> getTokenDictionary() {
-        return tokenDict;
-    }
-
-    @Override
-    public void reset() {
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java
deleted file mode 100644
index 8e36335..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/DoubleFieldValueGenerator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-public class DoubleFieldValueGenerator implements IFieldValueGenerator<Double> {
-    protected final Random rnd;
-
-    public DoubleFieldValueGenerator(Random rnd) {
-        this.rnd = rnd;
-    }
-
-    @Override
-    public Double next() {
-        return rnd.nextDouble();
-    }
-
-    @Override
-    public void reset() {
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java
deleted file mode 100644
index 851a0f4..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/FloatFieldValueGenerator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-public class FloatFieldValueGenerator implements IFieldValueGenerator<Float> {
-    protected final Random rnd;
-
-    public FloatFieldValueGenerator(Random rnd) {
-        this.rnd = rnd;
-    }
-
-    @Override
-    public Float next() {
-        return rnd.nextFloat();
-    }
-
-    @Override
-    public void reset() {
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
deleted file mode 100644
index dbd4bfc..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IFieldValueGenerator.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-public interface IFieldValueGenerator<T> {
-    public T next();
-    public void reset();
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
deleted file mode 100644
index 6c5f649..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/IntegerFieldValueGenerator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-public class IntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
-    protected final Random rnd;
-
-    public IntegerFieldValueGenerator(Random rnd) {
-        this.rnd = rnd;
-    }
-
-    @Override
-    public Integer next() {
-        return rnd.nextInt();
-    }
-
-    @Override
-    public void reset() {
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java
deleted file mode 100644
index 5af0b0f..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/PersonNameFieldValueGenerator.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.hyracks.util.MathUtil;
-
-public class PersonNameFieldValueGenerator implements IFieldValueGenerator<String> {
-    private final String FIRST_NAMES_FILE = "dist.all.first.cleaned";
-    private final String LAST_NAMES_FILE = "dist.all.last.cleaned";
-
-    private final Random rnd;
-    private final double middleInitialProb;
-    private final String letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
-
-    private List<String> firstNames = new ArrayList<String>();
-    private List<String> lastNames = new ArrayList<String>();
-
-    public PersonNameFieldValueGenerator(Random rnd, double middleInitialProb)
-            throws IOException {
-        this.rnd = rnd;
-        this.middleInitialProb = middleInitialProb;
-        initNames();
-    }
-
-    private void initNames() throws IOException {
-        String line;
-
-        // Read first names from data file.
-        InputStream firstNamesIn = this.getClass().getClassLoader().getResourceAsStream(FIRST_NAMES_FILE);
-        BufferedReader firstNamesReader = new BufferedReader(new InputStreamReader(firstNamesIn));
-        try {
-            while ((line = firstNamesReader.readLine()) != null) {
-                firstNames.add(line.trim());
-            }
-        } finally {
-            firstNamesReader.close();
-        }
-
-        // Read last names from data file.
-        InputStream lastNamesIn = this.getClass().getClassLoader().getResourceAsStream(LAST_NAMES_FILE);
-        BufferedReader lastNamesReader = new BufferedReader(new InputStreamReader(lastNamesIn));
-        try {
-            while ((line = lastNamesReader.readLine()) != null) {
-                lastNames.add(line.trim());
-            }
-        } finally {
-            lastNamesReader.close();
-        }
-    }
-
-    @Override
-    public String next() {
-        StringBuilder strBuilder = new StringBuilder();
-
-        // First name.
-        int fix = MathUtil.stripSignBit(rnd.nextInt()) % firstNames.size();
-        strBuilder.append(firstNames.get(fix));
-        strBuilder.append(" ");
-
-        // Optional middle initial.
-        double d = Math.abs(rnd.nextDouble());
-        if (d <= middleInitialProb) {
-            int mix = MathUtil.stripSignBit(rnd.nextInt()) % letters.length();
-            strBuilder.append(letters.charAt(mix));
-            strBuilder.append(". ");
-        }
-
-        // Last name.
-        int lix = MathUtil.stripSignBit(rnd.nextInt()) % lastNames.size();
-        strBuilder.append(lastNames.get(lix));
-
-        return strBuilder.toString();
-    }
-
-    @Override
-    public void reset() {
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java
deleted file mode 100644
index c7fbd3f..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/ProbabilityHelper.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Arrays;
-
-public class ProbabilityHelper {
-    public static double[] getUniformProbDist(int numChoices) {
-        double[] probDist = new double[numChoices];
-        for (int i = 0; i < numChoices; i++) {
-            probDist[i] = 1.0 / (double) numChoices;
-        }
-        return probDist;
-    }
-
-    public static double[] getZipfProbDist(int numChoices, int zipfSkew) {
-        double[] probDist = new double[numChoices];
-        double divisor = 0;
-        for (int i = 1; i <= numChoices; i++) {
-            divisor += 1.0 / (double) Math.pow((double) i, (double) zipfSkew);
-        }
-        for (int i = 1; i <= numChoices; i++) {
-            probDist[i - 1] = (1.0 / (double) Math.pow((double) i, (double) zipfSkew)) / divisor;
-        }
-        return probDist;
-    }
-
-    public static int[] getCumulIntRanges(double[] probDist) {
-        int[] opRanges = new int[probDist.length];
-        if (opRanges.length > 1) {
-            opRanges[0] = (int) Math.floor(Integer.MAX_VALUE * probDist[0]);
-            for (int i = 1; i < opRanges.length - 1; i++) {
-                opRanges[i] = opRanges[i - 1] + (int) Math.floor(Integer.MAX_VALUE * probDist[i]);
-            }
-            opRanges[opRanges.length - 1] = Integer.MAX_VALUE;
-        } else {
-            opRanges[0] = Integer.MAX_VALUE;
-        }
-        return opRanges;
-    }
-
-    public static int choose(int[] cumulIntRanges, int randomInt) {
-        int rndVal = Math.abs(randomInt);
-        int ix = Arrays.binarySearch(cumulIntRanges, rndVal);
-        if (ix < 0) {
-            ix = -ix - 1;
-        }
-        return ix;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java
deleted file mode 100644
index ddca6f3..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedDoubleFieldValueGenerator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-public class SortedDoubleFieldValueGenerator implements IFieldValueGenerator<Double> {
-    private double val;
-    private final double startVal;
-
-    public SortedDoubleFieldValueGenerator() {
-        startVal = 0.0d;
-        reset();
-    }
-
-    public SortedDoubleFieldValueGenerator(double startVal) {
-        this.startVal = startVal;
-        reset();
-    }
-
-    @Override
-    public Double next() {
-        return val++;
-    }
-
-    @Override
-    public void reset() {
-        val = startVal;
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java
deleted file mode 100644
index 1e5dd19..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedFloatFieldValueGenerator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-public class SortedFloatFieldValueGenerator implements IFieldValueGenerator<Float> {
-    private float val = 0.0f;
-    private final float startVal;
-
-    public SortedFloatFieldValueGenerator() {
-        startVal = 0.0f;
-        reset();
-    }
-
-    public SortedFloatFieldValueGenerator(float startVal) {
-        this.startVal = startVal;
-        reset();
-    }
-
-    @Override
-    public Float next() {
-        return val++;
-    }
-
-    @Override
-    public void reset() {
-        val = startVal;
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
deleted file mode 100644
index 3ab17ca..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/SortedIntegerFieldValueGenerator.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-public class SortedIntegerFieldValueGenerator implements IFieldValueGenerator<Integer> {
-    private int val = 0;
-    private final int startVal;
-
-    public SortedIntegerFieldValueGenerator() {
-        startVal = 0;
-        reset();
-    }
-
-    public SortedIntegerFieldValueGenerator(int startVal) {
-        this.startVal = startVal;
-        reset();
-    }
-
-    @Override
-    public Integer next() {
-        return val++;
-    }
-
-    @Override
-    public void reset() {
-        val = startVal;
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java
deleted file mode 100644
index 9418447..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/StringFieldValueGenerator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.util.Random;
-
-import org.apache.hyracks.util.MathUtil;
-
-public class StringFieldValueGenerator implements IFieldValueGenerator<String> {
-    private int maxLen;
-    private final Random rnd;
-
-    public StringFieldValueGenerator(int maxLen, Random rnd) {
-        this.maxLen = maxLen;
-        this.rnd = rnd;
-    }
-
-    public void setMaxLength(int maxLen) {
-        this.maxLen = maxLen;
-    }
-
-    @Override
-    public String next() {
-        String s = Long.toHexString(Double.doubleToLongBits(rnd.nextDouble()));
-        StringBuilder strBuilder = new StringBuilder();
-        for (int i = 0; i < s.length() && i < maxLen; i++) {
-            strBuilder.append(s.charAt(MathUtil.stripSignBit(rnd.nextInt()) % s.length()));
-        }
-        return strBuilder.toString();
-    }
-
-    @Override
-    public void reset() {
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java
deleted file mode 100644
index c34c7bc..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleBatch.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-
-@SuppressWarnings("rawtypes")
-public class TupleBatch {
-    private final int size;
-    private final TupleGenerator[] tupleGens;
-    public final AtomicBoolean inUse = new AtomicBoolean(false);
-
-    public TupleBatch(int size, IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
-        this.size = size;
-        tupleGens = new TupleGenerator[size];
-        for (int i = 0; i < size; i++) {
-            tupleGens[i] = new TupleGenerator(fieldGens, fieldSerdes, payloadSize);
-        }
-    }
-
-    public void generate() throws IOException {
-        for(TupleGenerator tupleGen : tupleGens) {
-            tupleGen.next();
-        }
-    }
-
-    public int size() {
-        return size;
-    }
-
-    public ITupleReference get(int ix) {
-        return tupleGens[ix].get();
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/5e17af20/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java
deleted file mode 100644
index 4f26065..0000000
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/datagen/TupleGenerator.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.datagen;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-
-@SuppressWarnings({"rawtypes", "unchecked" })
-public class TupleGenerator {
-    protected final ISerializerDeserializer[] fieldSerdes;
-    protected final IFieldValueGenerator[] fieldGens;
-    protected final ArrayTupleBuilder tb;
-    protected final ArrayTupleReference tuple;
-    protected final byte[] payload;
-    protected final DataOutput tbDos;
-
-    public TupleGenerator(IFieldValueGenerator[] fieldGens, ISerializerDeserializer[] fieldSerdes, int payloadSize) {
-        this.fieldSerdes = fieldSerdes;
-        this.fieldGens = fieldGens;
-        tuple = new ArrayTupleReference();
-        if (payloadSize > 0) {
-            tb = new ArrayTupleBuilder(fieldSerdes.length + 1);
-            payload = new byte[payloadSize];
-        } else {
-            tb = new ArrayTupleBuilder(fieldSerdes.length);
-            payload = null;
-        }
-        tbDos = tb.getDataOutput();
-    }
-
-    public ITupleReference next() throws IOException {
-        tb.reset();
-        for (int i = 0; i < fieldSerdes.length; i++) {
-            fieldSerdes[i].serialize(fieldGens[i].next(), tbDos);
-            tb.addFieldEndOffset();
-        }
-        if (payload != null) {
-            tbDos.write(payload);
-            tb.addFieldEndOffset();
-        }
-        tuple.reset(tb.getFieldEndOffsets(), tb.getByteArray());
-        return tuple;
-    }
-
-    public ITupleReference get() {
-        return tuple;
-    }
-
-    public void reset() {
-        for (IFieldValueGenerator fieldGen : fieldGens) {
-            fieldGen.reset();
-        }
-    }
-
-    public ISerializerDeserializer[] getFieldSerdes() {
-        return fieldSerdes;
-    }
-
-    public IFieldValueGenerator[] getFieldGens() {
-        return fieldGens;
-    }
-}


Mime
View raw message