asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject [3/8] asterixdb git commit: Move Twitter Firehose Datasource to Test Source Folder
Date Sat, 15 Oct 2016 18:01:57 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
deleted file mode 100644
index fcaa2a4..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/external/data/TweetGeneratorForSpatialIndexEvaluation.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.asterix.tools.external.data;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.InitializationInfo;
-import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessage;
-import org.apache.asterix.tools.external.data.DataGeneratorForSpatialIndexEvaluation.TweetMessageIterator;
-
-public class TweetGeneratorForSpatialIndexEvaluation {
-
-    private static Logger LOGGER = Logger.getLogger(TweetGeneratorForSpatialIndexEvaluation.class.getName());
-
-    public static final String KEY_DURATION = "duration";
-    public static final String KEY_TPS = "tps";
-    public static final String KEY_GUID_SEED = "guid-seed";
-    public static final String KEY_OPENSTREETMAP_FILEPATH = "open-street-map-filepath";
-    public static final String KEY_LOCATION_SAMPLE_INTERVAL = "location-sample-interval";
-
-    public static final String OUTPUT_FORMAT = "output-format";
-    public static final String OUTPUT_FORMAT_ARECORD = "arecord";
-    public static final String OUTPUT_FORMAT_ADM_STRING = "adm-string";
-
-    private static final int DEFAULT_DURATION = 60; //seconds
-    private static final int DEFAULT_GUID_SEED = 0;
-    private static final int DEFAULT_SAMPLE_INTERVAL = 1;
-
-    private int duration;
-    private TweetMessageIterator tweetIterator = null;
-    private int partition;
-    private int tweetCount = 0;
-    private int frameTweetCount = 0;
-    private int numFlushedTweets = 0;
-    private OutputStream os;
-    private DataGeneratorForSpatialIndexEvaluation dataGenerator = null;
-    private ByteBuffer outputBuffer = ByteBuffer.allocate(32 * 1024);
-    private GULongIDGenerator uidGenerator;
-    private String openStreetMapFilePath;
-    private int locationSampleInterval;
-
-    public int getTweetCount() {
-        return tweetCount;
-    }
-
-    public TweetGeneratorForSpatialIndexEvaluation(Map<String, String> configuration, int partition, String format,
-            OutputStream os) throws Exception {
-        this.partition = partition;
-        String value = configuration.get(KEY_DURATION);
-        this.duration = value != null ? Integer.parseInt(value) : DEFAULT_DURATION;
-        openStreetMapFilePath = configuration.get(KEY_OPENSTREETMAP_FILEPATH);
-        String lsi = configuration.get(KEY_LOCATION_SAMPLE_INTERVAL);
-        this.locationSampleInterval = lsi != null ? Integer.parseInt(lsi) : DEFAULT_SAMPLE_INTERVAL;
-        int guidSeed = configuration.get(KEY_GUID_SEED) != null ? Integer.parseInt(configuration.get(KEY_GUID_SEED))
-                : DEFAULT_GUID_SEED;
-        uidGenerator = new GULongIDGenerator(partition, (byte) (guidSeed));
-        dataGenerator = new DataGeneratorForSpatialIndexEvaluation(new InitializationInfo(), openStreetMapFilePath,
-                locationSampleInterval);
-        tweetIterator = dataGenerator.new TweetMessageIterator(duration, uidGenerator);
-        this.os = os;
-    }
-
-    private void writeTweetString(TweetMessage tweetMessage) throws IOException {
-        String tweet = tweetMessage.toString() + "\n";
-        tweetCount++;
-        byte[] b = tweet.getBytes();
-        if (outputBuffer.position() + b.length > outputBuffer.limit()) {
-            flush();
-            numFlushedTweets += frameTweetCount;
-            frameTweetCount = 0;
-            outputBuffer.put(b);
-        } else {
-            outputBuffer.put(b);
-        }
-        frameTweetCount++;
-    }
-
-    public int getNumFlushedTweets() {
-        return numFlushedTweets;
-    }
-
-    private void flush() throws IOException {
-        outputBuffer.flip();
-        os.write(outputBuffer.array(), 0, outputBuffer.limit());
-        outputBuffer.position(0);
-        outputBuffer.limit(32 * 1024);
-    }
-
-    public boolean setNextRecordBatch(int numTweetsInBatch) throws Exception {
-        boolean moreData = tweetIterator.hasNext();
-        if (!moreData) {
-            if (outputBuffer.position() > 0) {
-                flush();
-                numFlushedTweets += frameTweetCount;
-                frameTweetCount = 0;
-            }
-            if (LOGGER.isLoggable(Level.INFO)) {
-                LOGGER.info("Reached end of batch. Tweet Count: [" + partition + "]" + tweetCount);
-            }
-            return false;
-        } else {
-            int count = 0;
-            while (count < numTweetsInBatch) {
-                writeTweetString(tweetIterator.next());
-                count++;
-            }
-            return true;
-        }
-    }
-
-    public void resetDurationAndFlushedTweetCount(int duration) {
-        tweetIterator.resetDuration(duration);
-        numFlushedTweets = 0;
-        tweetCount = 0;
-
-    }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
deleted file mode 100644
index 26656f6..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/tbltoadm/TblToAdm.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.tools.tbltoadm;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.PrintWriter;
-
-public class TblToAdm {
-
-    private static final String SEPARATOR_STRING = "\\|\\s*";
-    private static final char SEPARATOR_CHAR = '|';
-
-    private static void convertFile(String inputFileName, String outputFileName, String scmFileName) throws IOException {
-        File scmFile = new File(scmFileName);
-        File inFile = new File(inputFileName);
-        File outFile = new File(outputFileName);
-
-        BufferedReader scmReader = new BufferedReader(new FileReader(scmFile));
-        String scmStr = scmReader.readLine();
-        String[] fieldInfo = scmStr.split(SEPARATOR_STRING);
-        String[] fieldNames = new String[fieldInfo.length];
-        boolean[] fieldIsString = new boolean[fieldInfo.length];
-        for (int i = 0; i < fieldInfo.length; i++) {
-            String[] f = fieldInfo[i].split(":\\s*");
-            fieldNames[i] = f[0];
-            fieldIsString[i] = f[1].equals("string");
-        }
-        scmReader.close();
-
-        BufferedReader reader = new BufferedReader(new FileReader(inFile));
-        PrintWriter writer = new PrintWriter(outFile);
-        String row;
-        while ((row = reader.readLine()) != null) {
-            // String[] fields = row.split(SEPARATOR);
-            if (row.length() == 0 || row.charAt(0) == ' ' || row.charAt(0) == '\n') {
-                continue;
-            }
-            writer.write("{  ");
-            int pos = 0;
-            int strlen = row.length();
-            for (int i = 0; i < fieldNames.length; i++) {
-                if (i > 0) {
-                    writer.write(", ");
-                }
-                writer.print('"');
-                writer.write(fieldNames[i]);
-                writer.write("\": ");
-                if (fieldIsString[i]) {
-                    writer.print('"');
-                }
-                for (; pos < strlen; pos++) {
-                    char c = row.charAt(pos);
-                    if (c != SEPARATOR_CHAR) {
-                        writer.print(c);
-
-                    } else {
-                        ++pos;
-                        break;
-                    }
-                }
-                if (fieldIsString[i]) {
-                    writer.print('"');
-                }
-            }
-            writer.write("  }\n");
-        }
-        writer.close();
-        reader.close();
-    }
-
-    public static void main(String args[]) throws Exception {
-        if (args.length != 3) {
-            System.err.println("Usage: bash tbl2adm <srcfile.tbl> <destfile.adm> <schema-file>");
-            return;
-        }
-        convertFile(args[0], args[1], args[2]);
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java b/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
deleted file mode 100644
index a525b1b..0000000
--- a/asterixdb/asterix-tools/src/main/java/org/apache/asterix/tools/translator/ADGenDmlTranslator.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.tools.translator;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.asterix.common.annotations.TypeDataGen;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.lang.common.statement.DataverseDecl;
-import org.apache.asterix.lang.common.statement.TypeDecl;
-import org.apache.asterix.metadata.MetadataException;
-import org.apache.asterix.metadata.MetadataTransactionContext;
-import org.apache.asterix.om.types.IAType;
-import org.apache.asterix.om.types.TypeSignature;
-import org.apache.asterix.translator.AbstractLangTranslator;
-import org.apache.asterix.translator.TypeTranslator;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-public class ADGenDmlTranslator extends AbstractLangTranslator {
-
-    private final MetadataTransactionContext mdTxnCtx;
-    private final List<Statement> aqlStatements;
-    private Map<TypeSignature, IAType> types;
-    private Map<TypeSignature, TypeDataGen> typeDataGenMap;
-
-    public ADGenDmlTranslator(MetadataTransactionContext mdTxnCtx, List<Statement> aqlStatements) {
-        this.mdTxnCtx = mdTxnCtx;
-        this.aqlStatements = aqlStatements;
-    }
-
-    public void translate() throws AsterixException, MetadataException, AlgebricksException {
-        String defaultDataverse = getDefaultDataverse();
-        types = new HashMap<TypeSignature, IAType>();
-        typeDataGenMap = new HashMap<TypeSignature, TypeDataGen>();
-
-        for (Statement stmt : aqlStatements) {
-            if (stmt.getKind() == Statement.Kind.TYPE_DECL) {
-                TypeDecl td = (TypeDecl) stmt;
-                String typeDataverse =
-                        td.getDataverseName() == null ? defaultDataverse : td.getDataverseName().getValue();
-
-                Map<TypeSignature, IAType> typeInStmt = TypeTranslator.computeTypes(mdTxnCtx, td.getTypeDef(),
-                        td.getIdent().getValue(), typeDataverse, types);
-                types.putAll(typeInStmt);
-
-                TypeSignature signature = new TypeSignature(typeDataverse, td.getIdent().getValue());
-                TypeDataGen tdg = td.getDatagenAnnotation();
-                if (tdg != null) {
-                    typeDataGenMap.put(signature, tdg);
-                }
-            }
-        }
-    }
-
-    private String getDefaultDataverse() {
-        for (Statement stmt : aqlStatements) {
-            if (stmt.getKind() == Statement.Kind.DATAVERSE_DECL) {
-                return ((DataverseDecl) stmt).getDataverseName().getValue();
-            }
-        }
-        return null;
-    }
-
-    public Map<TypeSignature, IAType> getTypeMap() {
-        return types;
-    }
-
-    public Map<TypeSignature, TypeDataGen> getTypeDataGenMap() {
-        return typeDataGenMap;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
new file mode 100644
index 0000000..8a1adb0
--- /dev/null
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.datagen;
+
+import java.io.File;
+
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.CmdLineParser;
+
+public class AdgClientDriver {
+
+    public static final int NUMBER_OF_ARGUMENTS = 2;
+
+    public static class AdgClientConfig {
+
+        @Argument(index = 0, required = true, metaVar = "ARG1", usage = "The file containing the annotated schema.")
+        private File schemaFile;
+
+        @Argument(index = 1, required = true, metaVar = "ARG2", usage = "The output directory path.")
+        private File outputDir;
+    }
+
+    public static void main(String[] args) throws Exception {
+        AdgClientConfig acc = new AdgClientConfig();
+        CmdLineParser cmdParser = new CmdLineParser(acc);
+        try {
+            cmdParser.parseArgument(args);
+        } catch (Exception e) {
+            cmdParser.printUsage(System.err);
+            throw e;
+        }
+        AdmDataGen adg = new AdmDataGen(acc.schemaFile, acc.outputDir);
+        adg.init();
+        adg.dataGen();
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
new file mode 100644
index 0000000..2918575
--- /dev/null
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
@@ -0,0 +1,1020 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.datagen;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.asterix.common.annotations.AutoDataGen;
+import org.apache.asterix.common.annotations.DateBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.DatetimeAddRandHoursDataGen;
+import org.apache.asterix.common.annotations.DatetimeBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.FieldIntervalDataGen;
+import org.apache.asterix.common.annotations.FieldValFileDataGen;
+import org.apache.asterix.common.annotations.FieldValFileSameIndexDataGen;
+import org.apache.asterix.common.annotations.IRecordFieldDataGen;
+import org.apache.asterix.common.annotations.IRecordTypeAnnotation;
+import org.apache.asterix.common.annotations.IRecordTypeAnnotation.Kind;
+import org.apache.asterix.common.annotations.InsertRandIntDataGen;
+import org.apache.asterix.common.annotations.ListDataGen;
+import org.apache.asterix.common.annotations.ListValFileDataGen;
+import org.apache.asterix.common.annotations.RecordDataGenAnnotation;
+import org.apache.asterix.common.annotations.TypeDataGen;
+import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.transactions.JobId;
+import org.apache.asterix.lang.aql.parser.AQLParserFactory;
+import org.apache.asterix.lang.aql.parser.ParseException;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.IParserFactory;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.metadata.MetadataException;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.TypeSignature;
+import org.apache.asterix.om.util.NonTaggedFormatUtil;
+import org.apache.asterix.tools.translator.ADGenDmlTranslator;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
+import org.apache.hyracks.algebricks.data.utils.WriteValueTools;
+
+public class AdmDataGen {
+
+    class DataGeneratorContext {
+        private Map<File, String[]> fileToLoadedDataMap = new HashMap<File, String[]>();
+
+        public Map<File, String[]> getFileToLoadedDataMap() {
+            return fileToLoadedDataMap;
+        }
+    }
+
+    public static void printDate(int year, int month, int day, PrintStream out) throws IOException {
+        WriteValueTools.writeInt(year, out);
+        out.print("-");
+        if (month < 10) {
+            out.print("0");
+        }
+        WriteValueTools.writeInt(month, out);
+        out.print("-");
+        if (day < 10) {
+            out.print("0");
+        }
+        WriteValueTools.writeInt(day, out);
+    }
+
+    abstract class AbstractValueGenerator {
+        protected PrintStream out;
+        protected DataGeneratorContext ctx;
+
+        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+            this.out = out;
+            this.ctx = ctx;
+        }
+
+        public abstract void generate() throws IOException;
+    }
+
+    abstract class RandomValueGenerator extends AbstractValueGenerator {
+        protected Random rnd;
+
+        @Override
+        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+            super.init(out, ctx);
+            this.rnd = new Random(51);
+        }
+
+    }
+
+    class IntAutoGenerator extends AbstractValueGenerator {
+
+        private final int initValue;
+        private int val;
+
+        public IntAutoGenerator(int initValue) {
+            this.initValue = initValue;
+        }
+
+        @Override
+        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+            super.init(out, ctx);
+            val = initValue;
+        }
+
+        @Override
+        public void generate() throws IOException {
+            WriteValueTools.writeInt(val, out);
+            val++;
+        }
+
+    }
+
+    class LongAutoGenerator extends AbstractValueGenerator {
+
+        private final long initValue;
+        private long val;
+
+        public LongAutoGenerator(long initValue) {
+            this.initValue = initValue;
+        }
+
+        @Override
+        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+            super.init(out, ctx);
+            val = initValue;
+        }
+
+        @Override
+        public void generate() throws IOException {
+            WriteValueTools.writeLong(val, out);
+            out.print("i64");
+            val++;
+        }
+
+    }
+
+    class StringFromArrayGenerator extends RandomValueGenerator {
+        private final String[][] array;
+        private int lastIndex;
+        private final String constructor;
+        private String[] chosen;
+
+        public StringFromArrayGenerator(String[][] array, String constructor) {
+            this.array = array;
+            this.constructor = constructor;
+        }
+
+        @Override
+        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+            super.init(out, ctx);
+            this.lastIndex = -1;
+            this.chosen = new String[array.length];
+        }
+
+        @Override
+        public void generate() throws IOException {
+            for (int i = 0; i < array.length; i++) {
+                lastIndex = Math.abs(rnd.nextInt()) % array[i].length;
+                chosen[i] = array[i][lastIndex];
+            }
+            if (constructor != null) {
+                out.print(constructor);
+                out.print("(");
+            }
+            out.print("\"");
+            for (int i = 0; i < chosen.length; i++) {
+                out.print(chosen[i]);
+            }
+            out.print("\"");
+            if (constructor != null) {
+                out.print(")");
+            }
+        }
+
+        public int getLastIndex() {
+            return lastIndex;
+        }
+    }
+
+    abstract class AbstractCollectionDataGenerator extends RandomValueGenerator {
+        protected final AbstractCollectionType act;
+        protected final int min;
+        protected final int max;
+        protected final String startList;
+        protected final String endList;
+
+        public AbstractCollectionDataGenerator(AbstractCollectionType act, int min, int max) {
+            this.act = act;
+            this.min = min;
+            this.max = max;
+            if (act.getTypeTag() == ATypeTag.ORDEREDLIST) {
+                startList = "[";
+                endList = "]";
+            } else {
+                startList = "{{";
+                endList = "}}";
+            }
+        }
+
+    }
+
+    class ListDataGenerator extends AbstractCollectionDataGenerator {
+
+        private AbstractValueGenerator itemGen;
+
+        public ListDataGenerator(AbstractCollectionType act, int min, int max) {
+            super(act, min, max);
+        }
+
+        @Override
+        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+            super.init(out, ctx);
+            IAType t = act.getItemType();
+            if (t.getTypeTag() != ATypeTag.RECORD) {
+                throw new NotImplementedException("list annotation only works with record item types for now.");
+            }
+            ARecordType rt = (ARecordType) t;
+            RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
+            if (dga == null) {
+                throw new Exception("No annotation on item type " + t);
+            }
+            itemGen = new RecordGenerator(rt, dga, null);
+            itemGen.init(out, ctx);
+        }
+
+        @Override
+        public void generate() throws IOException {
+            out.print(startList);
+            int numItems = rnd.nextInt(max - min + 1) + min;
+            for (int i = 0; i < numItems; i++) {
+                if (i > 0) {
+                    out.print(", ");
+                }
+                itemGen.generate();
+            }
+            out.print(endList);
+        }
+
+    }
+
+    class ListFromArrayGenerator extends AbstractCollectionDataGenerator {
+
+        private final String[] array;
+        private String constructor;
+
+        public ListFromArrayGenerator(AbstractCollectionType act, String[] array, int min, int max) {
+            super(act, min, max);
+            this.array = array;
+        }
+
+        @Override
+        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+            super.init(out, ctx);
+            if (act.getItemType().getTypeTag() == ATypeTag.STRING) {
+                constructor = null;
+            } else {
+                constructor = getConstructor(act.getItemType());
+            }
+        }
+
+        @Override
+        public void generate() throws IOException {
+            out.print(startList);
+            int numItems = rnd.nextInt(max - min + 1) + min;
+            for (int i = 0; i < numItems; i++) {
+                if (i > 0) {
+                    out.print(", ");
+                }
+                int c = Math.abs(rnd.nextInt()) % array.length;
+                if (constructor != null) {
+                    out.print(constructor);
+                    out.print("(");
+                }
+                out.print("\"");
+                out.print(array[c]);
+                out.print("\"");
+                if (constructor != null) {
+                    out.print(")");
+                }
+            }
+            out.print(endList);
+        }
+
+    }
+
+    class StringFromArraySameIndexGenerator extends AbstractValueGenerator {
+        private final String[] array;
+        private final StringFromArrayGenerator sfag;
+        private final String constructor;
+
+        public StringFromArraySameIndexGenerator(String[] array, StringFromArrayGenerator sfag, String constructor) {
+            this.array = array;
+            this.sfag = sfag;
+            this.constructor = constructor;
+        }
+
+        @Override
+        public void generate() throws IOException {
+            String val = array[sfag.getLastIndex()];
+            if (constructor != null) {
+                out.print(constructor);
+                out.print("(");
+            }
+            out.print("\"");
+            out.print(val);
+            out.print("\"");
+            if (constructor != null) {
+                out.print(")");
+            }
+        }
+    }
+
+    class IntIntervalGenerator extends RandomValueGenerator {
+
+        private final int min;
+        private final int max;
+        private final String prefix;
+        private final String suffix;
+
+        public IntIntervalGenerator(int min, int max, String prefix, String suffix) {
+            this.min = min;
+            this.max = max;
+            this.prefix = prefix;
+            this.suffix = suffix;
+        }
+
+        @Override
+        public void generate() throws IOException {
+            int v = Math.abs(rnd.nextInt() % (max - min + 1)) + min;
+            if (prefix != null) {
+                out.print(prefix);
+            }
+            WriteValueTools.writeInt(v, out);
+            if (suffix != null) {
+                out.print(suffix);
+            }
+        }
+
+    }
+
+    class LongIntervalGenerator extends RandomValueGenerator {
+
+        private final long min;
+        private final long max;
+        private final String prefix;
+        private final String suffix;
+
+        public LongIntervalGenerator(long min, long max, String prefix, String suffix) {
+            this.min = min;
+            this.max = max;
+            this.prefix = prefix;
+            this.suffix = suffix;
+        }
+
+        @Override
+        public void generate() throws IOException {
+            long v = Math.abs(rnd.nextLong() % (max - min + 1)) + min;
+            if (prefix != null) {
+                out.print(prefix);
+            }
+            WriteValueTools.writeLong(v, out);
+            if (suffix != null) {
+                out.print(suffix);
+            }
+        }
+
+    }
+
+    class DoubleIntervalGenerator extends RandomValueGenerator {
+
+        private final double min;
+        private final double max;
+        private final String prefix;
+        private final String suffix;
+
+        public DoubleIntervalGenerator(double min, double max, String prefix, String suffix) {
+            this.min = min;
+            this.max = max;
+            this.prefix = prefix;
+            this.suffix = suffix;
+        }
+
+        @Override
+        public void generate() throws IOException {
+            double v = Math.abs(rnd.nextDouble() % (max - min)) + min;
+            if (prefix != null) {
+                out.print(prefix);
+            }
+            out.print(v);
+            if (suffix != null) {
+                out.print(suffix);
+            }
+        }
+
+    }
+
+    class InsertRandIntGenerator extends RandomValueGenerator {
+
+        private final String str1;
+        private final String str2;
+
+        public InsertRandIntGenerator(String str1, String str2) {
+            this.str1 = "\"" + str1;
+            this.str2 = str2 + "\"";
+        }
+
+        @Override
+        public void generate() throws IOException {
+            int v = Math.abs(rnd.nextInt());
+            out.print(str1);
+            WriteValueTools.writeInt(v, out);
+            out.print(str2);
+        }
+
+    }
+
+    interface AccessibleDateGenerator {
+        public int getYear();
+
+        public int getMonth();
+
+        public int getDay();
+    }
+
+    abstract class DateGenerator extends RandomValueGenerator implements AccessibleDateGenerator {
+
+        private int year;
+        private int month;
+        private int day;
+
+        protected void generateDate(int minYear, int maxYear) throws IOException {
+            year = rnd.nextInt(maxYear - minYear + 1) + minYear;
+            month = Math.abs(rnd.nextInt()) % 12 + 1;
+            day = Math.abs(rnd.nextInt()) % 28 + 1;
+            printDate(year, month, day, out);
+        }
+
+        @Override
+        public int getYear() {
+            return year;
+        }
+
+        @Override
+        public int getMonth() {
+            return month;
+        }
+
+        @Override
+        public int getDay() {
+            return day;
+        }
+    }
+
+    class DateBetweenYearsGenerator extends DateGenerator {
+
+        private final int minYear;
+        private final int maxYear;
+
+        public DateBetweenYearsGenerator(int minYear, int maxYear) {
+            this.minYear = minYear;
+            this.maxYear = maxYear;
+        }
+
+        @Override
+        public void generate() throws IOException {
+            out.print("date(\"");
+            generateDate(minYear, maxYear);
+            out.print("\")");
+        }
+
+    }
+
+    interface AccessibleDatetimeGenerator extends AccessibleDateGenerator {
+        public int getHour();
+
+        public int getMinute();
+
+        public int getSecond();
+    }
+
+    class DatetimeBetweenYearsGenerator extends DateGenerator implements AccessibleDatetimeGenerator {
+        private final int minYear;
+        private final int maxYear;
+        private int hour;
+
+        public DatetimeBetweenYearsGenerator(int minYear, int maxYear) {
+            this.minYear = minYear;
+            this.maxYear = maxYear;
+        }
+
+        @Override
+        public void generate() throws IOException {
+            out.print("datetime(\"");
+            generateDate(minYear, maxYear);
+            out.print("T");
+            hour = rnd.nextInt(24);
+            if (hour < 10) {
+                out.print("0");
+            }
+            WriteValueTools.writeInt(hour, out);
+            out.print(":00:00\")");
+        }
+
+        @Override
+        public int getHour() {
+            return hour;
+        }
+
+        @Override
+        public int getMinute() {
+            return 0;
+        }
+
+        @Override
+        public int getSecond() {
+            return 0;
+        }
+
+    }
+
+    class DatetimeAddRandHoursGenerator extends RandomValueGenerator {
+
+        private final int minHours;
+        private final int maxHours;
+        private final AccessibleDatetimeGenerator adtg;
+
+        public DatetimeAddRandHoursGenerator(int minHours, int maxHours, AccessibleDatetimeGenerator adtg) {
+            this.minHours = minHours;
+            this.maxHours = maxHours;
+            this.adtg = adtg;
+        }
+
+        @Override
+        public void generate() throws IOException {
+            int addHours = rnd.nextInt(maxHours - minHours + 1) + minHours;
+
+            out.print("datetime(\"");
+            printDate(adtg.getYear(), adtg.getMonth(), adtg.getDay(), out);
+            out.print("T");
+            int h = adtg.getHour() + addHours;
+            if (h > 23) {
+                h = 23;
+            }
+            if (h < 10) {
+                out.print("0");
+            }
+            WriteValueTools.writeInt(h, out);
+            out.print(":");
+            int m = adtg.getMinute();
+            if (m < 10) {
+                out.print("0");
+            }
+            WriteValueTools.writeInt(m, out);
+            out.print(":");
+            int s = adtg.getSecond();
+            if (s < 10) {
+                out.print("0");
+            }
+            WriteValueTools.writeInt(s, out);
+            out.print("\")");
+        }
+
+    }
+
+    class GenFieldsIntGenerator extends RandomValueGenerator {
+        private final int minFields;
+        private final int maxFields;
+        private final String prefix;
+        private final int startIndex;
+        private String[] fieldNames;
+        private int[] id;
+        private int[] x;
+
+        public GenFieldsIntGenerator(int startIndex, int minFields, int maxFields, String prefix) {
+            this.startIndex = startIndex;
+            this.minFields = minFields;
+            this.maxFields = maxFields;
+            this.prefix = prefix;
+        }
+
+        @Override
+        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+            super.init(out, ctx);
+            fieldNames = new String[maxFields];
+            for (int i = 0; i < maxFields; i++) {
+                fieldNames[i] = prefix + "_" + i;
+            }
+            id = new int[maxFields];
+            for (int i = 0; i < maxFields; i++) {
+                id[i] = i;
+            }
+            x = new int[maxFields];
+        }
+
+        @Override
+        public void generate() throws IOException {
+            int numFields = rnd.nextInt(maxFields - minFields + 1) + minFields;
+            System.arraycopy(id, 0, x, 0, maxFields);
+            int n = numFields;
+            boolean first = startIndex < 1;
+            while (n > 0) {
+                int c = rnd.nextInt(n);
+                if (first) {
+                    first = false;
+                } else {
+                    out.print(",");
+                }
+                out.print(" \"");
+                out.print(fieldNames[x[c]]);
+                out.print("\": ");
+                WriteValueTools.writeInt(Math.abs(rnd.nextInt()), out);
+                x[c] = x[n - 1];
+                n--;
+            }
+        }
+    }
+
+    class RecordGenerator extends RandomValueGenerator {
+
+        private AbstractValueGenerator[] declaredFieldsGenerators;
+        private boolean[] nullable;
+        private AbstractValueGenerator undeclaredFieldsGenerator;
+        private final ARecordType recType;
+        private final RecordDataGenAnnotation annot;
+        private final String printAfter;
+
+        public RecordGenerator(ARecordType type, RecordDataGenAnnotation annot, String printAfter) {
+            this.recType = type;
+            this.annot = annot;
+            this.printAfter = printAfter;
+        }
+
+        @Override
+        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
+            super.init(out, ctx);
+            if (declaredFieldsGenerators == null) {
+                int m = annot.getDeclaredFieldsDatagen().length;
+                declaredFieldsGenerators = new AbstractValueGenerator[m];
+                nullable = new boolean[m];
+                for (int i = 0; i < m; i++) {
+                    IAType ti = recType.getFieldTypes()[i];
+                    if (NonTaggedFormatUtil.isOptional(ti)) {
+                        ti = ((AUnionType) ti).getActualType();
+                        nullable[i] = true;
+                    }
+                    IRecordFieldDataGen rfdg = annot.getDeclaredFieldsDatagen()[i];
+                    if (rfdg == null) {
+                        if (ti.getTypeTag() == ATypeTag.RECORD) {
+                            ARecordType rt = (ARecordType) ti;
+                            RecordDataGenAnnotation dga = null;
+                            for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
+                                if (annot.getKind() == Kind.RECORD_DATA_GEN) {
+                                    dga = (RecordDataGenAnnotation) annot;
+                                    break;
+                                }
+                            }
+                            if (dga != null) {
+                                declaredFieldsGenerators[i] = new RecordGenerator(rt, dga, null);
+                                continue;
+                            }
+                        }
+                        if (declaredFieldsGenerators[i] == null) {
+                            throw new Exception("No data generator annotation for field " + recType.getFieldNames()[i]
+                                    + " in type " + recType);
+                        }
+                    }
+                    switch (rfdg.getKind()) {
+                        case VALFILE: {
+                            FieldValFileDataGen vf = (FieldValFileDataGen) rfdg;
+                            int numFiles = vf.getFiles().length;
+                            String[][] a = new String[numFiles][];
+                            for (int k = 0; k < numFiles; k++) {
+                                File f = vf.getFiles()[k];
+                                a[k] = ctx.getFileToLoadedDataMap().get(f);
+                                if (a[k] == null) {
+                                    a[k] = readFileAsStringArray(f);
+                                    ctx.getFileToLoadedDataMap().put(f, a[k]);
+                                }
+                            }
+                            String constructor;
+                            if (ti.getTypeTag() == ATypeTag.STRING) {
+                                constructor = null;
+                            } else {
+                                constructor = getConstructor(ti);
+                            }
+                            declaredFieldsGenerators[i] = new StringFromArrayGenerator(a, constructor);
+                            break;
+                        }
+                        case LISTVALFILE: {
+                            ListValFileDataGen lvf = (ListValFileDataGen) rfdg;
+                            String[] a = ctx.getFileToLoadedDataMap().get(lvf.getFile());
+                            if (a == null) {
+                                a = readFileAsStringArray(lvf.getFile());
+                                ctx.getFileToLoadedDataMap().put(lvf.getFile(), a);
+                            }
+                            if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
+                                throw new Exception(
+                                        "list-val-file annotation cannot be used for field of type " + ti.getTypeTag());
+                            }
+                            AbstractCollectionType act = (AbstractCollectionType) ti;
+                            declaredFieldsGenerators[i] = new ListFromArrayGenerator(act, a, lvf.getMin(),
+                                    lvf.getMax());
+                            break;
+                        }
+                        case VALFILESAMEINDEX: {
+                            FieldValFileSameIndexDataGen vfsi = (FieldValFileSameIndexDataGen) rfdg;
+                            String[] a = ctx.getFileToLoadedDataMap().get(vfsi.getFile());
+                            if (a == null) {
+                                a = readFileAsStringArray(vfsi.getFile());
+                                ctx.getFileToLoadedDataMap().put(vfsi.getFile(), a);
+                            }
+                            StringFromArrayGenerator sfag = null;
+                            for (int j = 0; j < i; j++) {
+                                if (recType.getFieldNames()[j].equals(vfsi.getSameAsField())) {
+                                    if (declaredFieldsGenerators[j] instanceof StringFromArrayGenerator) {
+                                        sfag = (StringFromArrayGenerator) declaredFieldsGenerators[j];
+                                        break;
+                                    } else {
+                                        throw new Exception("Data generator for field " + recType.getFieldNames()[j]
+                                                + " is not based on values from a text file, as required by generator for field "
+                                                + recType.getFieldNames()[i]);
+                                    }
+                                }
+                            }
+                            if (sfag == null) {
+                                throw new Exception("Couldn't find field " + vfsi.getSameAsField() + " before field "
+                                        + recType.getFieldNames()[i]);
+                            }
+                            String constructor;
+                            if (ti.getTypeTag() == ATypeTag.STRING) {
+                                constructor = null;
+                            } else {
+                                constructor = getConstructor(ti);
+                            }
+                            declaredFieldsGenerators[i] = new StringFromArraySameIndexGenerator(a, sfag, constructor);
+                            break;
+                        }
+                        case INTERVAL: {
+                            FieldIntervalDataGen fi = (FieldIntervalDataGen) rfdg;
+                            String prefix = null;
+                            String suffix = null;
+                            if (ti.getTypeTag() == ATypeTag.STRING) {
+                                prefix = "\"";
+                                suffix = "\"";
+                            }
+                            switch (fi.getValueType()) {
+                                case INT: {
+                                    declaredFieldsGenerators[i] = new IntIntervalGenerator(
+                                            Integer.parseInt(fi.getMin()), Integer.parseInt(fi.getMax()), prefix,
+                                            suffix);
+                                    break;
+                                }
+                                case LONG: {
+                                    declaredFieldsGenerators[i] = new LongIntervalGenerator(Long.parseLong(fi.getMin()),
+                                            Long.parseLong(fi.getMax()), prefix, suffix);
+                                    break;
+                                }
+                                case DOUBLE: {
+                                    declaredFieldsGenerators[i] = new DoubleIntervalGenerator(
+                                            Double.parseDouble(fi.getMin()), Double.parseDouble(fi.getMax()), prefix,
+                                            suffix);
+                                    break;
+                                }
+                                default: {
+                                    throw new NotImplementedException();
+                                }
+                            }
+                            break;
+                        }
+                        case INSERTRANDINT: {
+                            InsertRandIntDataGen iri = (InsertRandIntDataGen) rfdg;
+                            declaredFieldsGenerators[i] = new InsertRandIntGenerator(iri.getStr1(), iri.getStr2());
+                            break;
+                        }
+                        case LIST: {
+                            ListDataGen l = (ListDataGen) rfdg;
+                            if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
+                                throw new Exception(
+                                        "list-val-file annotation cannot be used for field of type " + ti.getTypeTag());
+                            }
+                            AbstractCollectionType act = (AbstractCollectionType) ti;
+                            declaredFieldsGenerators[i] = new ListDataGenerator(act, l.getMin(), l.getMax());
+                            break;
+                        }
+                        case DATEBETWEENYEARS: {
+                            DateBetweenYearsDataGen dby = (DateBetweenYearsDataGen) rfdg;
+                            declaredFieldsGenerators[i] = new DateBetweenYearsGenerator(dby.getMinYear(),
+                                    dby.getMaxYear());
+                            break;
+                        }
+                        case DATETIMEBETWEENYEARS: {
+                            DatetimeBetweenYearsDataGen dtby = (DatetimeBetweenYearsDataGen) rfdg;
+                            declaredFieldsGenerators[i] = new DatetimeBetweenYearsGenerator(dtby.getMinYear(),
+                                    dtby.getMaxYear());
+                            break;
+                        }
+                        case DATETIMEADDRANDHOURS: {
+                            DatetimeAddRandHoursDataGen dtarh = (DatetimeAddRandHoursDataGen) rfdg;
+                            AccessibleDatetimeGenerator adtg = null;
+                            for (int j = 0; j < i; j++) {
+                                if (recType.getFieldNames()[j].equals(dtarh.getAddToField())) {
+                                    if (declaredFieldsGenerators[j] instanceof AccessibleDatetimeGenerator) {
+                                        adtg = (AccessibleDatetimeGenerator) declaredFieldsGenerators[j];
+                                        break;
+                                    } else {
+                                        throw new Exception("Data generator for field " + recType.getFieldNames()[j]
+                                                + " is not based on accessible datetime values, as required by generator for field "
+                                                + recType.getFieldNames()[i]);
+                                    }
+                                }
+                            }
+                            if (adtg == null) {
+                                throw new Exception("Couldn't find field " + dtarh.getAddToField() + " before field "
+                                        + recType.getFieldNames()[i]);
+                            }
+                            declaredFieldsGenerators[i] = new DatetimeAddRandHoursGenerator(dtarh.getMinHour(),
+                                    dtarh.getMaxHour(), adtg);
+                            break;
+                        }
+                        case AUTO: {
+                            AutoDataGen auto = (AutoDataGen) rfdg;
+                            switch (ti.getTypeTag()) {
+                                case INT32: {
+                                    declaredFieldsGenerators[i] = new IntAutoGenerator(
+                                            Integer.parseInt(auto.getInitValueStr()));
+                                    break;
+                                }
+                                case INT64: {
+                                    declaredFieldsGenerators[i] = new LongAutoGenerator(
+                                            Long.parseLong(auto.getInitValueStr()));
+                                    break;
+                                }
+                                default: {
+                                    throw new IllegalStateException(rfdg.getKind()
+                                            + " annotation is not implemented for type " + ti.getTypeTag());
+                                }
+                            }
+                            break;
+                        }
+                        default: {
+                            throw new NotImplementedException(rfdg.getKind() + " is not implemented");
+                        }
+                    }
+                }
+            }
+            for (int i = 0; i < declaredFieldsGenerators.length; i++) {
+                declaredFieldsGenerators[i].init(out, ctx);
+            }
+            if (undeclaredFieldsGenerator == null) {
+                UndeclaredFieldsDataGen ufdg = annot.getUndeclaredFieldsDataGen();
+                if (ufdg != null) {
+                    if (!recType.isOpen()) {
+                        throw new Exception("Cannot generate undeclared fields for closed type " + recType);
+                    }
+                    undeclaredFieldsGenerator = new GenFieldsIntGenerator(declaredFieldsGenerators.length,
+                            ufdg.getMinUndeclaredFields(), ufdg.getMaxUndeclaredFields(),
+                            ufdg.getUndeclaredFieldsPrefix());
+                }
+            }
+            if (undeclaredFieldsGenerator != null) {
+                undeclaredFieldsGenerator.init(out, ctx);
+            }
+        }
+
+        @Override
+        public void generate() throws IOException {
+            out.print("{");
+            boolean first = true;
+            for (int i = 0; i < declaredFieldsGenerators.length; i++) {
+                boolean pick;
+                if (nullable[i]) {
+                    pick = rnd.nextBoolean();
+                } else {
+                    pick = true;
+                }
+                if (pick) {
+                    if (first) {
+                        first = false;
+                    } else {
+                        out.print(",");
+                    }
+                    out.print(" \"");
+                    out.print(this.recType.getFieldNames()[i]);
+                    out.print("\": ");
+                    declaredFieldsGenerators[i].generate();
+                }
+            }
+            if (undeclaredFieldsGenerator != null) {
+                undeclaredFieldsGenerator.generate();
+            }
+            out.print(" }");
+            if (printAfter != null) {
+                out.print(printAfter);
+            }
+        }
+
+    }
+
+    private final File schemaFile;
+    private final File outputDir;
+    private Map<TypeSignature, IAType> typeMap;
+    private Map<TypeSignature, TypeDataGen> typeAnnotMap;
+    private DataGeneratorContext dgCtx;
+    private final IParserFactory parserFactory = new AQLParserFactory();
+
+    public AdmDataGen(File schemaFile, File outputDir) {
+        this.schemaFile = schemaFile;
+        this.outputDir = outputDir;
+    }
+
+    public void init() throws IOException, ParseException, AsterixException, ACIDException, MetadataException,
+            AlgebricksException {
+        FileReader aql = new FileReader(schemaFile);
+        IParser parser = parserFactory.createParser(aql);
+        List<Statement> statements = parser.parse();
+        aql.close();
+        // TODO: Need to fix how to use transactions here.
+        MetadataTransactionContext mdTxnCtx = new MetadataTransactionContext(new JobId(-1));
+        ADGenDmlTranslator dmlt = new ADGenDmlTranslator(mdTxnCtx, statements);
+        dmlt.translate();
+        typeMap = dmlt.getTypeMap();
+        typeAnnotMap = dmlt.getTypeDataGenMap();
+        dgCtx = new DataGeneratorContext();
+    }
+
+    public void dataGen() throws Exception {
+        for (Map.Entry<TypeSignature, IAType> me : typeMap.entrySet()) {
+            TypeSignature tn = me.getKey();
+            TypeDataGen tdg = typeAnnotMap.get(tn);
+            if (tdg.isDataGen()) {
+                IAType t = me.getValue();
+
+                if (t.getTypeTag() != ATypeTag.RECORD) {
+                    throw new NotImplementedException();
+                }
+                ARecordType rt = (ARecordType) t;
+                RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
+                if (dga == null) {
+                    throw new Exception("No data generator annotations for type " + tn);
+                }
+                File outFile = new File(outputDir + File.separator + tdg.getOutputFileName());
+                PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outFile)));
+                RecordGenerator rg = new RecordGenerator(rt, dga, "\n");
+                rg.init(outStream, dgCtx);
+                for (long i = 0; i < tdg.getNumValues(); i++) {
+                    rg.generate();
+                }
+                outStream.close();
+            }
+        }
+    }
+
+    private static RecordDataGenAnnotation firstDataGenAnnotation(ARecordType rt) {
+        for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
+            if (annot.getKind() == Kind.RECORD_DATA_GEN) {
+                return (RecordDataGenAnnotation) annot;
+            }
+        }
+        return null;
+    }
+
+    private static String[] readFileAsStringArray(File file) throws IOException {
+        List<String> tmp = new ArrayList<String>();
+        FileInputStream fstream = new FileInputStream(file);
+        DataInputStream in = new DataInputStream(fstream);
+        BufferedReader br = new BufferedReader(new InputStreamReader(in));
+        String strLine;
+        while ((strLine = br.readLine()) != null) {
+            tmp.add(strLine);
+        }
+        in.close();
+        return tmp.toArray(new String[0]);
+    }
+
+    private static String getConstructor(IAType t) throws Exception {
+        if (t instanceof BuiltinType) {
+            String s = ((BuiltinType) t).getConstructor();
+            if (s == null) {
+                throw new Exception("Type " + t + " has no constructors.");
+            }
+            return s;
+        } else {
+            throw new Exception("No string constructor for type " + t);
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java
new file mode 100644
index 0000000..b605f73
--- /dev/null
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/CustOrdDataGen.java
@@ -0,0 +1,476 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.datagen;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
+public class CustOrdDataGen {
+
+    // for customers
+    private static final String FIRST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.first.cleaned";
+    private static final String LAST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.last.cleaned";
+    private static final int MIN_AGE = 10;
+    private static final int MAX_AGE = 90;
+
+    private static final String[] STREETS = { "Main St.", "Oak St.", "7th St.", "Washington St.", "Cedar St.",
+            "Lake St.", "Hill St.", "Park St.", "View St." };
+    private static final int MIN_STREET_NUM = 1;
+    private static final int MAX_STREET_NUM = 10000;
+    private static final String[] CITIES = { "Seattle", "San Jose", "Mountain View", "Los Angeles", "Sunnyvale",
+            "Portland" };
+
+    private static final int MIN_INTERESTS = 0;
+    private static final int MAX_INTERESTS = 5;
+    private String[] INTERESTS = { "Bass", "Music", "Databases", "Fishing", "Tennis", "Squash", "Computers", "Books",
+            "Movies", "Cigars", "Wine", "Running", "Walking", "Skiing", "Basketball", "Video Games", "Cooking",
+            "Coffee", "Base Jumping", "Puzzles", "Chess", "Programming", "Reddit", "Soccer", "Hockey", "Money",
+            "Dancing", "Brewing", "Gardening", "Hacking", "Reading" };
+
+    private static final int MIN_CHILD_AGE = 0;
+
+    private static final int MIN_CHILDREN = 0;
+    private static final int MAX_CHILDREN = 5;
+
+    // for orders
+    private static final int MIN_ORDERS_PER_CUST = 1;
+    private static final int MAX_ORDERS_PER_CUST = 10;
+
+    private String[] CLERKS = { "Kathrin", "Katherine", "Cathryn", "Catherine", "Cat", "Kathryne", "Cathrin" };
+    private String[] ORDER_PRIORITIES = { "LOW", "MEDIUM", "HIGH", "PREMIUM" };
+    private String[] ORDER_STATUSES = { "ORDER_PLACED", "PAYMENT_RECEIVED", "ORDER_SHIPPED", "ORDER_DELIVERED" };
+
+    private String[] firstNames = { "Joe", "John", "Jill", "Gill", "Bill", "William", "Kathy", "Cathey", "Jane",
+            "Albert" };
+    private String[] lastNames = { "Doe", "Smith", "Li", "Singh", "Williams", "Davis", "Brown", "Wilson", "Moore",
+            "Thomas" };
+
+    private static final String[] UNDECLARED_FIELD_NAMES = { "param1", "param2", "param3", "param4", "param5",
+            "param6", "param7", "param8", "param9", "param10" };
+
+    private int currentCID = 0;
+    private int currentOID = 0;
+
+    private Random rndValue = new Random(50);
+
+    private Random rndChildAgeField = new Random(50);
+    private Random rndCustAgeField = new Random(50);
+    private Random rndCustAddressField = new Random(50);
+
+    private Random[] rndUndeclaredOrderFields;
+
+    private class Child {
+        public String name;
+        public int age;
+
+        public void generateFieldValues(String lastName, int maxChildAge) {
+            // name
+            int firstNameIx = Math.abs(rndValue.nextInt()) % firstNames.length;
+            name = firstNames[firstNameIx] + " " + lastName;
+
+            // age
+            age = -1;
+            if (rndChildAgeField.nextBoolean()) {
+                if (maxChildAge >= 0) {
+                    if (maxChildAge == MIN_CHILD_AGE) {
+                        age = maxChildAge;
+                    } else {
+                        age = Math.abs((rndValue.nextInt()) % (maxChildAge - MIN_CHILD_AGE)) + MIN_AGE;
+                    }
+                }
+            }
+        }
+
+        public String getJSON() {
+            StringBuilder jsonString = new StringBuilder();
+
+            jsonString.append("{ "); // start child
+
+            // name
+            jsonString.append(" \"name\": ");
+            jsonString.append("\"" + name + "\"");
+
+            // age
+            if (age >= 0) {
+                jsonString.append(", ");
+                jsonString.append(" \"age\": ");
+                jsonString.append(age);
+            }
+
+            jsonString.append(" }"); // end child
+
+            return jsonString.toString();
+        }
+
+    }
+
+    private class Customer {
+
+        private int cid;
+        private String name;
+        private int age;
+
+        private String streetName;
+        private int streetNumber;
+        private String city;
+
+        private int[] custInterests;
+        private Child[] custChildren;
+
+        public void generateFieldValues() {
+            cid = currentCID++;
+
+            int firstNameIx = Math.abs(rndValue.nextInt()) % firstNames.length;
+            int lastNameIx = Math.abs(rndValue.nextInt()) % lastNames.length;
+            name = firstNames[firstNameIx] + " " + lastNames[lastNameIx];
+
+            if (rndCustAgeField.nextBoolean()) {
+                age = Math.abs((rndValue.nextInt()) % (MAX_AGE - MIN_AGE)) + MIN_AGE;
+            } else {
+                age = -1;
+            }
+
+            if (rndCustAddressField.nextBoolean()) {
+                streetNumber = Math.abs(rndValue.nextInt()) % (MAX_STREET_NUM - MIN_STREET_NUM) + MIN_STREET_NUM;
+
+                int streetIx = Math.abs(rndValue.nextInt()) % STREETS.length;
+                streetName = STREETS[streetIx];
+
+                int cityIx = Math.abs(rndValue.nextInt()) % CITIES.length;
+                city = CITIES[cityIx];
+            } else {
+                streetNumber = -1;
+                streetName = null;
+                city = null;
+            }
+
+            int numInterests = Math.abs((rndValue.nextInt()) % (MAX_INTERESTS - MIN_INTERESTS)) + MIN_INTERESTS;
+            custInterests = new int[numInterests];
+            for (int i = 0; i < numInterests; i++) {
+                custInterests[i] = Math.abs(rndValue.nextInt()) % INTERESTS.length;
+            }
+
+            int numChildren = Math.abs((rndValue.nextInt()) % (MAX_CHILDREN - MIN_CHILDREN)) + MIN_CHILDREN;
+            custChildren = new Child[numChildren];
+            for (int i = 0; i < numChildren; i++) {
+                Child c = new Child();
+                int maxChildAge = age <= 0 ? 50 : (age - 20);
+                c.generateFieldValues(lastNames[lastNameIx], maxChildAge);
+                custChildren[i] = c;
+            }
+        }
+
+        public String getJSON() {
+            StringBuilder jsonString = new StringBuilder();
+
+            jsonString.append("{ "); // start customer
+
+            // customer id
+            jsonString.append(" \"cid\": ");
+            jsonString.append(cid);
+            jsonString.append(", ");
+
+            // name
+            jsonString.append(" \"name\": ");
+            jsonString.append("\"" + name + "\"");
+
+            // age
+            if (age >= 0) {
+                jsonString.append(", ");
+                jsonString.append(" \"age\": ");
+                jsonString.append(age);
+            }
+
+            // nested address
+            if (streetNumber >= 0) {
+                jsonString.append(", ");
+                jsonString.append(" \"address\": ");
+                jsonString.append("{ "); // start address
+
+                // number
+                jsonString.append(" \"number\": ");
+                jsonString.append(streetNumber);
+                jsonString.append(", ");
+
+                // street
+                jsonString.append(" \"street\": ");
+                jsonString.append("\"" + streetName + "\"");
+                jsonString.append(", ");
+
+                // city
+                jsonString.append(" \"city\": ");
+                jsonString.append("\"" + city + "\"");
+
+                jsonString.append(" }"); // end address
+            }
+
+            jsonString.append(", ");
+
+            // interests
+            jsonString.append(" \"interests\": ");
+            jsonString.append("{{ "); // start interests
+            for (int i = 0; i < custInterests.length; i++) {
+                jsonString.append("\"" + INTERESTS[custInterests[i]] + "\"");
+                if (i != custInterests.length - 1) {
+                    jsonString.append(", ");
+                }
+            }
+            jsonString.append(" }}"); // end interests
+            jsonString.append(", ");
+
+            // children
+            jsonString.append(" \"children\": ");
+            jsonString.append("[ "); // start children
+            for (int i = 0; i < custChildren.length; i++) {
+                String jsonChild = custChildren[i].getJSON();
+                jsonString.append(jsonChild);
+                if (i != custChildren.length - 1) {
+                    jsonString.append(", ");
+                }
+            }
+            jsonString.append(" ]"); // end children
+
+            jsonString.append(" }"); // end customer
+
+            return jsonString.toString();
+        }
+    }
+
+    private class Order {
+        public int oid;
+        public int cid;
+        public String orderStatus;
+        public String orderPriority;
+        public String clerk;
+        public String total;
+
+        private int[] undeclaredFields;
+        private BitSet nullMap;
+
+        public void generateFieldValues(Customer cust) {
+            cid = cust.cid;
+            oid = currentOID++;
+            float t = Math.abs(rndValue.nextFloat()) * 100;
+            total = t + "f";
+
+            int orderStatusIx = Math.abs(rndValue.nextInt()) % ORDER_STATUSES.length;
+            orderStatus = ORDER_STATUSES[orderStatusIx];
+
+            int orderPriorityIx = Math.abs(rndValue.nextInt()) % ORDER_PRIORITIES.length;
+            orderPriority = ORDER_PRIORITIES[orderPriorityIx];
+
+            int clerkIx = Math.abs(rndValue.nextInt()) % CLERKS.length;
+            clerk = CLERKS[clerkIx];
+
+            int m = rndUndeclaredOrderFields.length;
+            undeclaredFields = new int[m];
+            nullMap = new BitSet(m);
+            for (int i = 0; i < m; i++) {
+                if (rndUndeclaredOrderFields[i].nextBoolean()) {
+                    undeclaredFields[i] = rndValue.nextInt();
+                } else {
+                    nullMap.set(i);
+                }
+            }
+        }
+
+        public String getJSON() {
+            StringBuilder jsonString = new StringBuilder();
+
+            jsonString.append("{ "); // start order
+
+            // oid
+            jsonString.append(" \"oid\": ");
+            jsonString.append(oid);
+            jsonString.append(", ");
+
+            // cid
+            jsonString.append(" \"cid\": ");
+            jsonString.append(cid);
+            jsonString.append(", ");
+
+            // orderStatus
+            jsonString.append(" \"orderstatus\": ");
+            jsonString.append("\"" + orderStatus + "\"");
+            jsonString.append(", ");
+
+            // orderPriority
+            jsonString.append(" \"orderpriority\": ");
+            jsonString.append("\"" + orderPriority + "\"");
+            jsonString.append(", ");
+
+            // clerk
+            jsonString.append(" \"clerk\": ");
+            jsonString.append("\"" + clerk + "\"");
+            jsonString.append(", ");
+
+            // / cid
+            jsonString.append(" \"total\": ");
+            jsonString.append(total);
+
+            for (int i = 0; i < undeclaredFields.length; i++) {
+                if (!nullMap.get(i)) {
+                    jsonString.append(", ");
+                    jsonString.append(" \"" + UNDECLARED_FIELD_NAMES[i] + "\": ");
+                    jsonString.append(undeclaredFields[i]);
+                }
+            }
+
+            jsonString.append(" }"); // end order
+            return jsonString.toString();
+        }
+    }
+
+    public void init() {
+        try {
+            List<String> tmpFirstNames = new ArrayList<String>();
+
+            FileInputStream fstream = new FileInputStream(FIRST_NAMES_FILE_NAME);
+            DataInputStream in = new DataInputStream(fstream);
+            BufferedReader br = new BufferedReader(new InputStreamReader(in));
+            String strLine;
+            while ((strLine = br.readLine()) != null) {
+                String firstLetter = strLine.substring(0, 1);
+                String remainder = strLine.substring(1);
+                String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+                tmpFirstNames.add(capitalized);
+            }
+            in.close();
+            firstNames = tmpFirstNames.toArray(firstNames);
+
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
+        try {
+            List<String> tmpLastNames = new ArrayList<String>();
+
+            FileInputStream fstream = new FileInputStream(LAST_NAMES_FILE_NAME);
+            DataInputStream in = new DataInputStream(fstream);
+            BufferedReader br = new BufferedReader(new InputStreamReader(in));
+            String strLine;
+            while ((strLine = br.readLine()) != null) {
+                String firstLetter = strLine.substring(0, 1);
+                String remainder = strLine.substring(1);
+                String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+                tmpLastNames.add(capitalized);
+            }
+            in.close();
+            lastNames = tmpLastNames.toArray(firstNames);
+
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
+        rndUndeclaredOrderFields = new Random[UNDECLARED_FIELD_NAMES.length];
+        for (int i = 0; i < rndUndeclaredOrderFields.length; i++) {
+            rndUndeclaredOrderFields[i] = new Random(50);
+        }
+    }
+
+    public void writeOrdersList(List<Order> ordersList, FileWriter ordersFile) throws IOException {
+        while (!ordersList.isEmpty()) {
+            int ix = Math.abs(rndValue.nextInt()) % ordersList.size();
+            ordersFile.write(ordersList.get(ix).getJSON() + "\n");
+            ordersList.remove(ix);
+        }
+    }
+
+    public void writeCustomerList(List<Customer> customerList, Order[] ordersBatch, List<Order> ordersList,
+            FileWriter customersFile, FileWriter ordersFile) throws IOException {
+        while (!customerList.isEmpty()) {
+            int ix = Math.abs(rndValue.nextInt()) % customerList.size();
+            customersFile.write(customerList.get(ix).getJSON() + "\n");
+
+            // generate orders
+            int numOrders = Math.abs(rndValue.nextInt()) % (MAX_ORDERS_PER_CUST - MIN_ORDERS_PER_CUST)
+                    + MIN_ORDERS_PER_CUST;
+            for (int i = 0; i < numOrders; i++) {
+                ordersBatch[i].generateFieldValues(customerList.get(ix));
+                ordersList.add(ordersBatch[i]);
+            }
+            writeOrdersList(ordersList, ordersFile);
+
+            customerList.remove(ix);
+        }
+    }
+
+    public static void main(String[] args) throws IOException {
+        if (args.length != 2) {
+            System.err.println("MUST PROVIDE 2 PARAMS, 1. output dir name and 2. number of records to generate.");
+            System.exit(1);
+        }
+
+        String outputFile = args[0];
+        int numRecords = Integer.parseInt(args[1]);
+
+        FileWriter customersFile = new FileWriter(outputFile + File.separator + "customer.adm");
+        FileWriter ordersFile = new FileWriter(outputFile + File.separator + "orders.adm");
+
+        CustOrdDataGen dataGen = new CustOrdDataGen();
+        dataGen.init();
+
+        int batchSize = 1000;
+        Customer[] customerBatch = new Customer[batchSize];
+        for (int i = 0; i < batchSize; i++) {
+            customerBatch[i] = dataGen.new Customer();
+        }
+
+        Order[] ordersBatch = new Order[MAX_ORDERS_PER_CUST];
+        for (int i = 0; i < MAX_ORDERS_PER_CUST; i++) {
+            ordersBatch[i] = dataGen.new Order();
+        }
+
+        List<Customer> customerList = new LinkedList<Customer>();
+        List<Order> ordersList = new LinkedList<Order>();
+        int custIx = 0;
+        for (int i = 0; i < numRecords; i++) {
+
+            customerBatch[custIx].generateFieldValues();
+            customerList.add(customerBatch[custIx]);
+            custIx++;
+
+            if (customerList.size() >= batchSize) {
+                dataGen.writeCustomerList(customerList, ordersBatch, ordersList, customersFile, ordersFile);
+                custIx = 0;
+            }
+        }
+        dataGen.writeCustomerList(customerList, ordersBatch, ordersList, customersFile, ordersFile);
+        customersFile.flush();
+        customersFile.close();
+
+        ordersFile.flush();
+        ordersFile.close();
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cb92dad7/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/EventDataGen.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/EventDataGen.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/EventDataGen.java
new file mode 100644
index 0000000..b175b42
--- /dev/null
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/EventDataGen.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.tools.datagen;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public class EventDataGen {
+    private static final String FIRST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.first.cleaned";
+    private static final String LAST_NAMES_FILE_NAME = "/opt/us_census_names/dist.all.last.cleaned";
+
+    private String[] firstNames = new String[10];
+    private String[] lastNames = new String[10];
+
+    private static final int MIN_USER_INTERESTS = 0;
+    private static final int MAX_USER_INTERESTS = 7;
+    private String[] INTERESTS = { "bass", "music", "databases", "fishing", "tennis", "squash", "computers", "books",
+            "movies", "cigars", "wine", "running", "walking", "skiing", "basketball", "video games", "cooking",
+            "coffee", "base jumping", "puzzles" };
+
+    private static final String[] STREETS = { "Main St.", "Oak St.", "7th St.", "Washington St.", "Cedar St.",
+            "Lake St.", "Hill St.", "Park St.", "View St." };
+    private static final int MIN_STREET_NUM = 1;
+    private static final int MAX_STREET_NUM = 10000;
+    private static final String[] CITIES = { "Seattle", "Irvine", "Laguna Beach", "Los Angeles", "San Clemente",
+            "Huntington Beach", "Portland" };
+    private static final int MIN_ZIP = 100000;
+    private static final int MAX_ZIP = 999999;
+    private static final String[] LAT_LONGS = { "47,-122", "33,-117", "33,-117", "34,-118", "33,-117", "33,-117",
+            "45,-122" };
+
+    private static final int MIN_MEMBERSHIPS = 1;
+    private static final int MAX_MEMBERSHIPS = 10;
+    private static final int MIN_SIG_ID = 1;
+    private static final int MAX_SIG_ID = 100;
+    private static final String[] CHAPTER_NAMES = { "Seattle", "Irvine", "Laguna Beach", "Los Angeles", "San Clemente",
+            "Huntington Beach", "Portland", "Newport Beach", "Kirkland" };
+    private static final int MEMBER_SINCE_MIN_YEAR = 1970;
+    private static final int MEMBER_SINCE_MAX_YEAR = 1998;
+
+    private Random rndValue = new Random(50);
+    private User user;
+
+    private final class User {
+        private int firstNameIdx;
+        private int lastNameIdx;
+        private int[] interests = new int[MAX_USER_INTERESTS - MIN_USER_INTERESTS];
+        int numInterests;
+        private int streetNumber;
+        private String street;
+        private String city;
+        private int zip;
+        private String latlong;
+        int numMemberships;
+        private int[] member_sigid = new int[MAX_MEMBERSHIPS];
+        private String[] member_chap_name = new String[MAX_MEMBERSHIPS];
+        private String[] member_since_date = new String[MAX_MEMBERSHIPS];
+
+        public void generateFieldValues() {
+            firstNameIdx = Math.abs(rndValue.nextInt()) % firstNames.length;
+            lastNameIdx = Math.abs(rndValue.nextInt()) % lastNames.length;
+            // name = firstNames[firstNameIx] + " " + lastNames[lastNameIx];
+            numInterests = Math.abs((rndValue.nextInt()) % (MAX_USER_INTERESTS - MIN_USER_INTERESTS))
+                    + MIN_USER_INTERESTS;
+            for (int i = 0; i < numInterests; i++) {
+                interests[i] = Math.abs(rndValue.nextInt()) % INTERESTS.length;
+            }
+            streetNumber = Math.abs(rndValue.nextInt()) % (MAX_STREET_NUM - MIN_STREET_NUM) + MIN_STREET_NUM;
+            street = STREETS[Math.abs(rndValue.nextInt()) % STREETS.length];
+            int cityIdx = Math.abs(rndValue.nextInt()) % CITIES.length;
+            city = CITIES[cityIdx];
+            zip = Math.abs(rndValue.nextInt() % (MAX_ZIP - MIN_ZIP)) + MIN_ZIP;
+            latlong = LAT_LONGS[cityIdx];
+            numMemberships = Math.abs(rndValue.nextInt()) % (MAX_MEMBERSHIPS - MIN_MEMBERSHIPS) + MIN_MEMBERSHIPS;
+            for (int i = 0; i < numMemberships; i++) {
+                member_sigid[i] = Math.abs(rndValue.nextInt()) % (MAX_SIG_ID - MIN_SIG_ID) + MIN_SIG_ID;
+                int cnIdx = Math.abs(rndValue.nextInt()) % CHAPTER_NAMES.length;
+                member_chap_name[i] = CHAPTER_NAMES[cnIdx];
+                int msYear = Math.abs(rndValue.nextInt()) % (MEMBER_SINCE_MAX_YEAR - MEMBER_SINCE_MIN_YEAR)
+                        + MEMBER_SINCE_MIN_YEAR;
+                int msMo = Math.abs(rndValue.nextInt()) % 12 + 1;
+                int msDay = Math.abs(rndValue.nextInt()) % 28 + 1;
+                member_since_date[i] = msYear + "-" + (msMo < 10 ? "0" : "") + msMo + "-" + (msDay < 10 ? "0" : "")
+                        + msDay;
+            }
+        }
+
+        public void write(Writer writer) throws IOException {
+            writer.append("{");
+            writer.append(" \"name\": \"");
+            writer.append(firstNames[firstNameIdx]);
+            writer.append(" ");
+            writer.append(lastNames[lastNameIdx]);
+            writer.append("\", ");
+
+            writer.append(" \"email\": \"");
+            writer.append(firstNames[firstNameIdx]);
+            writer.append(".");
+            writer.append(lastNames[lastNameIdx]);
+            writer.append("@example.com\", ");
+
+            writer.append(" \"interests\": <");
+            for (int i = 0; i < numInterests; i++) {
+                if (i > 0) {
+                    writer.append(", ");
+                }
+                writer.append("\"");
+                writer.append(INTERESTS[interests[i]]);
+                writer.append("\"");
+            }
+            writer.append(">, ");
+
+            writer.append(" \"address\": {");
+            writer.append(" \"street\": \"");
+            writer.append(streetNumber + " " + street);
+            writer.append("\",");
+            writer.append(" \"city\": \"");
+            writer.append(city);
+            writer.append("\",");
+            writer.append(" \"zip\": \"");
+            writer.append(zip + "\",");
+            writer.append(" \"latlong\": point(\"");
+            writer.append(latlong);
+            writer.append("\")");
+            writer.append("}, ");
+
+            writer.append(" \"member_of\": <");
+            for (int i = 0; i < numMemberships; i++) {
+                if (i > 0) {
+                    writer.append(", ");
+                }
+                writer.append("{");
+                writer.append(" \"sig_id\": ");
+                writer.append(member_sigid[i] + ",");
+                writer.append(" \"chapter_name\": \"");
+                writer.append(member_chap_name[i]);
+                writer.append("\",");
+                writer.append(" \"member_since\": date(\"");
+                writer.append(member_since_date[i]);
+                writer.append("\") }");
+            }
+            writer.append(">");
+
+            writer.append(" }\n");
+        }
+    }
+
+    public void init() throws IOException {
+        {
+            List<String> tmpFirstNames = new ArrayList<String>();
+
+            FileInputStream fstream = new FileInputStream(FIRST_NAMES_FILE_NAME);
+            DataInputStream in = new DataInputStream(fstream);
+            BufferedReader br = new BufferedReader(new InputStreamReader(in));
+            String strLine;
+            while ((strLine = br.readLine()) != null) {
+                String firstLetter = strLine.substring(0, 1);
+                String remainder = strLine.substring(1);
+                String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+                tmpFirstNames.add(capitalized);
+            }
+            in.close();
+            firstNames = tmpFirstNames.toArray(firstNames);
+        }
+        {
+            List<String> tmpLastNames = new ArrayList<String>();
+
+            FileInputStream fstream = new FileInputStream(LAST_NAMES_FILE_NAME);
+            DataInputStream in = new DataInputStream(fstream);
+            BufferedReader br = new BufferedReader(new InputStreamReader(in));
+            String strLine;
+            while ((strLine = br.readLine()) != null) {
+                String firstLetter = strLine.substring(0, 1);
+                String remainder = strLine.substring(1);
+                String capitalized = firstLetter.toUpperCase() + remainder.toLowerCase();
+                tmpLastNames.add(capitalized);
+            }
+            in.close();
+            lastNames = tmpLastNames.toArray(firstNames);
+        }
+        user = new User();
+    }
+
+    public void generate() {
+        user.generateFieldValues();
+    }
+
+    public void write(Writer w) throws IOException {
+        user.write(w);
+    }
+
+    public static void main(String[] args) throws IOException {
+        if (args.length != 2) {
+            System.err
+                    .println("MUST PROVIDE 2 PARAMETERS, 1. output directory path and 2. number of records to generate.");
+            System.exit(1);
+        }
+        String outputFile = args[0];
+        int numRecords = Integer.parseInt(args[1]);
+        Writer userFile = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputFile + File.separator
+                + "user.adm")));
+        EventDataGen dgen = new EventDataGen();
+        dgen.init();
+        for (int i = 0; i < numRecords; i++) {
+            dgen.generate();
+            dgen.write(userFile);
+        }
+        userFile.close();
+    }
+}


Mime
View raw message