asterix-tools/src/main/java/edu/uci/ics/asterix/tools/datagen/AdmDataGen.java

package edu.uci.ics.asterix.tools.datagen;

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;

import edu.uci.ics.asterix.aql.expression.Query;
import edu.uci.ics.asterix.aql.parser.AQLParser;
import edu.uci.ics.asterix.aql.parser.ParseException;
import edu.uci.ics.asterix.common.annotations.AutoDataGen;
import edu.uci.ics.asterix.common.annotations.DateBetweenYearsDataGen;
import edu.uci.ics.asterix.common.annotations.DatetimeAddRandHoursDataGen;
import edu.uci.ics.asterix.common.annotations.DatetimeBetweenYearsDataGen;
import edu.uci.ics.asterix.common.annotations.FieldIntervalDataGen;
import edu.uci.ics.asterix.common.annotations.FieldValFileDataGen;
import edu.uci.ics.asterix.common.annotations.FieldValFileSameIndexDataGen;
import edu.uci.ics.asterix.common.annotations.IRecordFieldDataGen;
import edu.uci.ics.asterix.common.annotations.IRecordTypeAnnotation;
import edu.uci.ics.asterix.common.annotations.IRecordTypeAnnotation.Kind;
import edu.uci.ics.asterix.common.annotations.InsertRandIntDataGen;
import edu.uci.ics.asterix.common.annotations.ListDataGen;
import edu.uci.ics.asterix.common.annotations.ListValFileDataGen;
import edu.uci.ics.asterix.common.annotations.RecordDataGenAnnotation;
import edu.uci.ics.asterix.common.annotations.TypeDataGen;
import edu.uci.ics.asterix.common.annotations.UndeclaredFieldsDataGen;
import edu.uci.ics.asterix.metadata.MetadataException;
import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
import edu.uci.ics.asterix.metadata.declared.AqlCompiledMetadataDeclarations;
import edu.uci.ics.asterix.om.types.ARecordType;
import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.asterix.om.types.AUnionType;
import edu.uci.ics.asterix.om.types.AbstractCollectionType;
import edu.uci.ics.asterix.om.types.BuiltinType;
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.asterix.tools.translator.ADGenDmlTranslator;
import edu.uci.ics.asterix.transaction.management.exception.ACIDException;
import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.core.utils.WriteValueTools;

public class AdmDataGen {

    class DataGeneratorContext {
        private Map<File, String[]> fileToLoadedDataMap = new HashMap<File, String[]>();

        public Map<File, String[]> getFileToLoadedDataMap() {
            return fileToLoadedDataMap;
        }
    }

    public static void printDate(int year, int month, int day, PrintStream out) throws IOException {
        WriteValueTools.writeInt(year, out);
        out.print("-");
        if (month < 10) {
            out.print("0");
        }
        WriteValueTools.writeInt(month, out);
        out.print("-");
        if (day < 10) {
            out.print("0");
        }
        WriteValueTools.writeInt(day, out);
    }

    abstract class AbstractValueGenerator {
        protected PrintStream out;
        protected DataGeneratorContext ctx;

        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
            this.out = out;
            this.ctx = ctx;
        }

        public abstract void generate() throws IOException;
    }

    abstract class RandomValueGenerator extends AbstractValueGenerator {
        protected Random rnd;

        @Override
        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
            super.init(out, ctx);
            this.rnd = new Random(51);
        }

    }

    class IntAutoGenerator extends AbstractValueGenerator {

        private final int initValue;
        private int val;

        public IntAutoGenerator(int initValue) {
            this.initValue = initValue;
        }

        @Override
        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
            super.init(out, ctx);
            val = initValue;
        }

        @Override
        public void generate() throws IOException {
            WriteValueTools.writeInt(val, out);
            val++;
        }

    }

    class LongAutoGenerator extends AbstractValueGenerator {

        private final long initValue;
        private long val;

        public LongAutoGenerator(long initValue) {
            this.initValue = initValue;
        }

        @Override
        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
            super.init(out, ctx);
            val = initValue;
        }

        @Override
        public void generate() throws IOException {
            WriteValueTools.writeLong(val, out);
            out.print("i64");
            val++;
        }

    }

    class StringFromArrayGenerator extends RandomValueGenerator {
        private final String[][] array;
        private int lastIndex;
        private final String constructor;
        private String[] chosen;

        public StringFromArrayGenerator(String[][] array, String constructor) {
            this.array = array;
            this.constructor = constructor;
        }

        @Override
        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
            super.init(out, ctx);
            this.lastIndex = -1;
            this.chosen = new String[array.length];
        }

        @Override
        public void generate() throws IOException {
            for (int i = 0; i < array.length; i++) {
                lastIndex = Math.abs(rnd.nextInt()) % array[i].length;
                chosen[i] = array[i][lastIndex];
            }
            if (constructor != null) {
                out.print(constructor);
                out.print("(");
            }
            out.print("\"");
            for (int i = 0; i < chosen.length; i++) {
                out.print(chosen[i]);
            }
            out.print("\"");
            if (constructor != null) {
                out.print(")");
            }
        }

        public int getLastIndex() {
            return lastIndex;
        }
    }

    abstract class AbstractCollectionDataGenerator extends RandomValueGenerator {
        protected final AbstractCollectionType act;
        protected final int min;
        protected final int max;
        protected final String startList;
        protected final String endList;

        public AbstractCollectionDataGenerator(AbstractCollectionType act, int min, int max) {
            this.act = act;
            this.min = min;
            this.max = max;
            if (act.getTypeTag() == ATypeTag.ORDEREDLIST) {
                startList = "[";
                endList = "]";
            } else {
                startList = "{{";
                endList = "}}";
            }
        }

    }

    class ListDataGenerator extends AbstractCollectionDataGenerator {

        private AbstractValueGenerator itemGen;

        public ListDataGenerator(AbstractCollectionType act, int min, int max) {
            super(act, min, max);
        }

        @Override
        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
            super.init(out, ctx);
            IAType t = act.getItemType();
            if (t.getTypeTag() != ATypeTag.RECORD) {
                throw new NotImplementedException("list annotation only works with record item types for now.");
            }
            ARecordType rt = (ARecordType) t;
            RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
            if (dga == null) {
                throw new Exception("No annotation on item type " + t);
            }
            itemGen = new RecordGenerator(rt, dga, null);
            itemGen.init(out, ctx);
        }

        @Override
        public void generate() throws IOException {
            out.print(startList);
            int numItems = rnd.nextInt(max - min + 1) + min;
            for (int i = 0; i < numItems; i++) {
                if (i > 0) {
                    out.print(", ");
                }
                itemGen.generate();
            }
            out.print(endList);
        }

    }

    class ListFromArrayGenerator extends AbstractCollectionDataGenerator {

        private final String[] array;
        private String constructor;

        public ListFromArrayGenerator(AbstractCollectionType act, String[] array, int min, int max) {
            super(act, min, max);
            this.array = array;
        }

        @Override
        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
            super.init(out, ctx);
            if (act.getItemType().getTypeTag() == ATypeTag.STRING) {
                constructor = null;
            } else {
                constructor = getConstructor(act.getItemType());
            }
        }

        @Override
        public void generate() throws IOException {
            out.print(startList);
            int numItems = rnd.nextInt(max - min + 1) + min;
            for (int i = 0; i < numItems; i++) {
                if (i > 0) {
                    out.print(", ");
                }
                int c = Math.abs(rnd.nextInt()) % array.length;
                if (constructor != null) {
                    out.print(constructor);
                    out.print("(");
                }
                out.print("\"");
                out.print(array[c]);
                out.print("\"");
                if (constructor != null) {
                    out.print(")");
                }
            }
            out.print(endList);
        }

    }

    class StringFromArraySameIndexGenerator extends AbstractValueGenerator {
        private final String[] array;
        private final StringFromArrayGenerator sfag;
        private final String constructor;

        public StringFromArraySameIndexGenerator(String[] array, StringFromArrayGenerator sfag, String constructor) {
            this.array = array;
            this.sfag = sfag;
            this.constructor = constructor;
        }

        @Override
        public void generate() throws IOException {
            String val = array[sfag.getLastIndex()];
            if (constructor != null) {
                out.print(constructor);
                out.print("(");
            }
            out.print("\"");
            out.print(val);
            out.print("\"");
            if (constructor != null) {
                out.print(")");
            }
        }
    }

    class IntIntervalGenerator extends RandomValueGenerator {

        private final int min;
        private final int max;
        private final String prefix;
        private final String suffix;

        public IntIntervalGenerator(int min, int max, String prefix, String suffix) {
            this.min = min;
            this.max = max;
            this.prefix = prefix;
            this.suffix = suffix;
        }

        @Override
        public void generate() throws IOException {
            int v = Math.abs(rnd.nextInt() % (max - min + 1)) + min;
            if (prefix != null) {
                out.print(prefix);
            }
            WriteValueTools.writeInt(v, out);
            if (suffix != null) {
                out.print(suffix);
            }
        }

    }

    class LongIntervalGenerator extends RandomValueGenerator {

        private final long min;
        private final long max;
        private final String prefix;
        private final String suffix;

        public LongIntervalGenerator(long min, long max, String prefix, String suffix) {
            this.min = min;
            this.max = max;
            this.prefix = prefix;
            this.suffix = suffix;
        }

        @Override
        public void generate() throws IOException {
            long v = Math.abs(rnd.nextLong() % (max - min + 1)) + min;
            if (prefix != null) {
                out.print(prefix);
            }
            WriteValueTools.writeLong(v, out);
            if (suffix != null) {
                out.print(suffix);
            }
        }

    }

    class DoubleIntervalGenerator extends RandomValueGenerator {

        private final double min;
        private final double max;
        private final String prefix;
        private final String suffix;

        public DoubleIntervalGenerator(double min, double max, String prefix, String suffix) {
            this.min = min;
            this.max = max;
            this.prefix = prefix;
            this.suffix = suffix;
        }

        @Override
        public void generate() throws IOException {
            double v = Math.abs(rnd.nextDouble() % (max - min)) + min;
            if (prefix != null) {
                out.print(prefix);
            }
            out.print(v);
            if (suffix != null) {
                out.print(suffix);
            }
        }

    }

    class InsertRandIntGenerator extends RandomValueGenerator {

        private final String str1;
        private final String str2;

        public InsertRandIntGenerator(String str1, String str2) {
            this.str1 = "\"" + str1;
            this.str2 = str2 + "\"";
        }

        @Override
        public void generate() throws IOException {
            int v = Math.abs(rnd.nextInt());
            out.print(str1);
            WriteValueTools.writeInt(v, out);
            out.print(str2);
        }

    }

    interface AccessibleDateGenerator {
        public int getYear();

        public int getMonth();

        public int getDay();
    }

    abstract class DateGenerator extends RandomValueGenerator implements AccessibleDateGenerator {

        private int year;
        private int month;
        private int day;

        protected void generateDate(int minYear, int maxYear) throws IOException {
            year = rnd.nextInt(maxYear - minYear + 1) + minYear;
            month = Math.abs(rnd.nextInt()) % 12 + 1;
            day = Math.abs(rnd.nextInt()) % 28 + 1;
            printDate(year, month, day, out);
        }

        @Override
        public int getYear() {
            return year;
        }

        @Override
        public int getMonth() {
            return month;
        }

        @Override
        public int getDay() {
            return day;
        }
    }

    class DateBetweenYearsGenerator extends DateGenerator {

        private final int minYear;
        private final int maxYear;

        public DateBetweenYearsGenerator(int minYear, int maxYear) {
            this.minYear = minYear;
            this.maxYear = maxYear;
        }

        @Override
        public void generate() throws IOException {
            out.print("date(\"");
            generateDate(minYear, maxYear);
            out.print("\")");
        }

    }

    interface AccessibleDatetimeGenerator extends AccessibleDateGenerator {
        public int getHour();

        public int getMinute();

        public int getSecond();
    }

    class DatetimeBetweenYearsGenerator extends DateGenerator implements AccessibleDatetimeGenerator {
        private final int minYear;
        private final int maxYear;
        private int hour;

        public DatetimeBetweenYearsGenerator(int minYear, int maxYear) {
            this.minYear = minYear;
            this.maxYear = maxYear;
        }

        @Override
        public void generate() throws IOException {
            out.print("datetime(\"");
            generateDate(minYear, maxYear);
            out.print("T");
            hour = rnd.nextInt(24);
            if (hour < 10) {
                out.print("0");
            }
            WriteValueTools.writeInt(hour, out);
            out.print(":00:00\")");
        }

        @Override
        public int getHour() {
            return hour;
        }

        @Override
        public int getMinute() {
            return 0;
        }

        @Override
        public int getSecond() {
            return 0;
        }

    }

    class DatetimeAddRandHoursGenerator extends RandomValueGenerator {

        private final int minHours;
        private final int maxHours;
        private final AccessibleDatetimeGenerator adtg;

        public DatetimeAddRandHoursGenerator(int minHours, int maxHours, AccessibleDatetimeGenerator adtg) {
            this.minHours = minHours;
            this.maxHours = maxHours;
            this.adtg = adtg;
        }

        @Override
        public void generate() throws IOException {
            int addHours = rnd.nextInt(maxHours - minHours + 1) + minHours;

            out.print("datetime(\"");
            printDate(adtg.getYear(), adtg.getMonth(), adtg.getDay(), out);
            out.print("T");
            int h = adtg.getHour() + addHours;
            if (h > 23) {
                h = 23;
            }
            if (h < 10) {
                out.print("0");
            }
            WriteValueTools.writeInt(h, out);
            out.print(":");
            int m = adtg.getMinute();
            if (m < 10) {
                out.print("0");
            }
            WriteValueTools.writeInt(m, out);
            out.print(":");
            int s = adtg.getSecond();
            if (s < 10) {
                out.print("0");
            }
            WriteValueTools.writeInt(s, out);
            out.print("\")");
        }

    }

    class GenFieldsIntGenerator extends RandomValueGenerator {
        private final int minFields;
        private final int maxFields;
        private final String prefix;
        private final int startIndex;
        private String[] fieldNames;
        private int[] id;
        private int[] x;

        public GenFieldsIntGenerator(int startIndex, int minFields, int maxFields, String prefix) {
            this.startIndex = startIndex;
            this.minFields = minFields;
            this.maxFields = maxFields;
            this.prefix = prefix;
        }

        @Override
        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
            super.init(out, ctx);
            fieldNames = new String[maxFields];
            for (int i = 0; i < maxFields; i++) {
                fieldNames[i] = prefix + "_" + i;
            }
            id = new int[maxFields];
            for (int i = 0; i < maxFields; i++) {
                id[i] = i;
            }
            x = new int[maxFields];
        }

        @Override
        public void generate() throws IOException {
            int numFields = rnd.nextInt(maxFields - minFields + 1) + minFields;
            System.arraycopy(id, 0, x, 0, maxFields);
            int n = numFields;
            boolean first = startIndex < 1;
            while (n > 0) {
                int c = rnd.nextInt(n);
                if (first) {
                    first = false;
                } else {
                    out.print(",");
                }
                out.print(" \"");
                out.print(fieldNames[x[c]]);
                out.print("\": ");
                WriteValueTools.writeInt(Math.abs(rnd.nextInt()), out);
                x[c] = x[n - 1];
                n--;
            }
        }
    }

    class RecordGenerator extends RandomValueGenerator {

        private AbstractValueGenerator[] declaredFieldsGenerators;
        private boolean[] nullable;
        private AbstractValueGenerator undeclaredFieldsGenerator;
        private final ARecordType recType;
        private final RecordDataGenAnnotation annot;
        private final String printAfter;

        public RecordGenerator(ARecordType type, RecordDataGenAnnotation annot, String printAfter) {
            this.recType = type;
            this.annot = annot;
            this.printAfter = printAfter;
        }

        @Override
        public void init(PrintStream out, DataGeneratorContext ctx) throws Exception {
            super.init(out, ctx);
            if (declaredFieldsGenerators == null) {
                int m = annot.getDeclaredFieldsDatagen().length;
                declaredFieldsGenerators = new AbstractValueGenerator[m];
                nullable = new boolean[m];
                for (int i = 0; i < m; i++) {
                    IAType ti = recType.getFieldTypes()[i];
                    if (ti.getTypeTag() == ATypeTag.UNION) {
                        AUnionType ut = (AUnionType) ti;
                        if (ut.isNullableType()) {
                            ti = ut.getUnionList().get(1);
                            nullable[i] = true;
                        }
                    }
                    IRecordFieldDataGen rfdg = annot.getDeclaredFieldsDatagen()[i];
                    if (rfdg == null) {
                        if (ti.getTypeTag() == ATypeTag.RECORD) {
                            ARecordType rt = (ARecordType) ti;
                            RecordDataGenAnnotation dga = null;
                            for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
                                if (annot.getKind() == Kind.RECORD_DATA_GEN) {
                                    dga = (RecordDataGenAnnotation) annot;
                                    break;
                                }
                            }
                            if (dga != null) {
                                declaredFieldsGenerators[i] = new RecordGenerator(rt, dga, null);
                                continue;
                            }
                        }
                        if (declaredFieldsGenerators[i] == null) {
                            throw new Exception("No data generator annotation for field " + recType.getFieldNames()[i]
                                    + " in type " + recType);
                        }
                    }
                    switch (rfdg.getKind()) {
                        case VALFILE: {
                            FieldValFileDataGen vf = (FieldValFileDataGen) rfdg;
                            int numFiles = vf.getFiles().length;
                            String[][] a = new String[numFiles][];
                            for (int k = 0; k < numFiles; k++) {
                                File f = vf.getFiles()[k];
                                a[k] = ctx.getFileToLoadedDataMap().get(f);
                                if (a[k] == null) {
                                    a[k] = readFileAsStringArray(f);
                                    ctx.getFileToLoadedDataMap().put(f, a[k]);
                                }
                            }
                            String constructor;
                            if (ti.getTypeTag() == ATypeTag.STRING) {
                                constructor = null;
                            } else {
                                constructor = getConstructor(ti);
                            }
                            declaredFieldsGenerators[i] = new StringFromArrayGenerator(a, constructor);
                            break;
                        }
                        case LISTVALFILE: {
                            ListValFileDataGen lvf = (ListValFileDataGen) rfdg;
                            String[] a = ctx.getFileToLoadedDataMap().get(lvf.getFile());
                            if (a == null) {
                                a = readFileAsStringArray(lvf.getFile());
                                ctx.getFileToLoadedDataMap().put(lvf.getFile(), a);
                            }
                            if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
                                throw new Exception("list-val-file annotation cannot be used for field of type "
                                        + ti.getTypeTag());
                            }
                            AbstractCollectionType act = (AbstractCollectionType) ti;
                            declaredFieldsGenerators[i] = new ListFromArrayGenerator(act, a, lvf.getMin(), lvf.getMax());
                            break;
                        }
                        case VALFILESAMEINDEX: {
                            FieldValFileSameIndexDataGen vfsi = (FieldValFileSameIndexDataGen) rfdg;
                            String[] a = ctx.getFileToLoadedDataMap().get(vfsi.getFile());
                            if (a == null) {
                                a = readFileAsStringArray(vfsi.getFile());
                                ctx.getFileToLoadedDataMap().put(vfsi.getFile(), a);
                            }
                            StringFromArrayGenerator sfag = null;
                            for (int j = 0; j < i; j++) {
                                if (recType.getFieldNames()[j].equals(vfsi.getSameAsField())) {
                                    if (declaredFieldsGenerators[j] instanceof StringFromArrayGenerator) {
                                        sfag = (StringFromArrayGenerator) declaredFieldsGenerators[j];
                                        break;
                                    } else {
                                        throw new Exception(
                                                "Data generator for field "
                                                        + recType.getFieldNames()[j]
                                                        + " is not based on values from a text file, as required by generator for field "
                                                        + recType.getFieldNames()[i]);
                                    }
                                }
                            }
                            if (sfag == null) {
                                throw new Exception("Couldn't find field " + vfsi.getSameAsField() + " before field "
                                        + recType.getFieldNames()[i]);
                            }
                            String constructor;
                            if (ti.getTypeTag() == ATypeTag.STRING) {
                                constructor = null;
                            } else {
                                constructor = getConstructor(ti);
                            }
                            declaredFieldsGenerators[i] = new StringFromArraySameIndexGenerator(a, sfag, constructor);
                            break;
                        }
                        case INTERVAL: {
                            FieldIntervalDataGen fi = (FieldIntervalDataGen) rfdg;
                            String prefix = null;
                            String suffix = null;
                            if (ti.getTypeTag() == ATypeTag.STRING) {
                                prefix = "\"";
                                suffix = "\"";
                            }
                            switch (fi.getValueType()) {
                                case INT: {
                                    declaredFieldsGenerators[i] = new IntIntervalGenerator(
                                            Integer.parseInt(fi.getMin()), Integer.parseInt(fi.getMax()), prefix,
                                            suffix);
                                    break;
                                }
                                case LONG: {
                                    declaredFieldsGenerators[i] = new LongIntervalGenerator(
                                            Long.parseLong(fi.getMin()), Long.parseLong(fi.getMax()), prefix, suffix);
                                    break;
                                }
                                case DOUBLE: {
                                    declaredFieldsGenerators[i] = new DoubleIntervalGenerator(Double.parseDouble(fi
                                            .getMin()), Double.parseDouble(fi.getMax()), prefix, suffix);
                                    break;
                                }
                                default: {
                                    throw new NotImplementedException();
                                }
                            }
                            break;
                        }
                        case INSERTRANDINT: {
                            InsertRandIntDataGen iri = (InsertRandIntDataGen) rfdg;
                            declaredFieldsGenerators[i] = new InsertRandIntGenerator(iri.getStr1(), iri.getStr2());
                            break;
                        }
                        case LIST: {
                            ListDataGen l = (ListDataGen) rfdg;
                            if (ti.getTypeTag() != ATypeTag.ORDEREDLIST && ti.getTypeTag() != ATypeTag.UNORDEREDLIST) {
                                throw new Exception("list-val-file annotation cannot be used for field of type "
                                        + ti.getTypeTag());
                            }
                            AbstractCollectionType act = (AbstractCollectionType) ti;
                            declaredFieldsGenerators[i] = new ListDataGenerator(act, l.getMin(), l.getMax());
                            break;
                        }
                        case DATEBETWEENYEARS: {
                            DateBetweenYearsDataGen dby = (DateBetweenYearsDataGen) rfdg;
                            declaredFieldsGenerators[i] = new DateBetweenYearsGenerator(dby.getMinYear(),
                                    dby.getMaxYear());
                            break;
                        }
                        case DATETIMEBETWEENYEARS: {
                            DatetimeBetweenYearsDataGen dtby = (DatetimeBetweenYearsDataGen) rfdg;
                            declaredFieldsGenerators[i] = new DatetimeBetweenYearsGenerator(dtby.getMinYear(),
                                    dtby.getMaxYear());
                            break;
                        }
                        case DATETIMEADDRANDHOURS: {
                            DatetimeAddRandHoursDataGen dtarh = (DatetimeAddRandHoursDataGen) rfdg;
                            AccessibleDatetimeGenerator adtg = null;
                            for (int j = 0; j < i; j++) {
                                if (recType.getFieldNames()[j].equals(dtarh.getAddToField())) {
                                    if (declaredFieldsGenerators[j] instanceof AccessibleDatetimeGenerator) {
                                        adtg = (AccessibleDatetimeGenerator) declaredFieldsGenerators[j];
                                        break;
                                    } else {
                                        throw new Exception(
                                                "Data generator for field "
                                                        + recType.getFieldNames()[j]
                                                        + " is not based on accessible datetime values, as required by generator for field "
                                                        + recType.getFieldNames()[i]);
                                    }
                                }
                            }
                            if (adtg == null) {
                                throw new Exception("Couldn't find field " + dtarh.getAddToField() + " before field "
                                        + recType.getFieldNames()[i]);
                            }
                            declaredFieldsGenerators[i] = new DatetimeAddRandHoursGenerator(dtarh.getMinHour(),
                                    dtarh.getMaxHour(), adtg);
                            break;
                        }
                        case AUTO: {
                            AutoDataGen auto = (AutoDataGen) rfdg;
                            switch (ti.getTypeTag()) {
                                case INT32: {
                                    declaredFieldsGenerators[i] = new IntAutoGenerator(Integer.parseInt(auto
                                            .getInitValueStr()));
                                    break;
                                }
                                case INT64: {
                                    declaredFieldsGenerators[i] = new LongAutoGenerator(Long.parseLong(auto
                                            .getInitValueStr()));
                                    break;
                                }
                                default: {
                                    throw new IllegalStateException(rfdg.getKind()
                                            + " annotation is not implemented for type " + ti.getTypeTag());
                                }
                            }
                            break;
                        }
                        default: {
                            throw new NotImplementedException(rfdg.getKind() + " is not implemented");
                        }
                    }
                }
            }
            for (int i = 0; i < declaredFieldsGenerators.length; i++) {
                declaredFieldsGenerators[i].init(out, ctx);
            }
            if (undeclaredFieldsGenerator == null) {
                UndeclaredFieldsDataGen ufdg = annot.getUndeclaredFieldsDataGen();
                if (ufdg != null) {
                    if (!recType.isOpen()) {
                        throw new Exception("Cannot generate undeclared fields for closed type " + recType);
                    }
                    undeclaredFieldsGenerator = new GenFieldsIntGenerator(declaredFieldsGenerators.length,
                            ufdg.getMinUndeclaredFields(), ufdg.getMaxUndeclaredFields(),
                            ufdg.getUndeclaredFieldsPrefix());
                }
            }
            if (undeclaredFieldsGenerator != null) {
                undeclaredFieldsGenerator.init(out, ctx);
            }
        }

        @Override
        public void generate() throws IOException {
            out.print("{");
            boolean first = true;
            for (int i = 0; i < declaredFieldsGenerators.length; i++) {
                boolean pick;
                if (nullable[i]) {
                    pick = rnd.nextBoolean();
                } else {
                    pick = true;
                }
                if (pick) {
                    if (first) {
                        first = false;
                    } else {
                        out.print(",");
                    }
                    out.print(" \"");
                    out.print(this.recType.getFieldNames()[i]);
                    out.print("\": ");
                    declaredFieldsGenerators[i].generate();
                }
            }
            if (undeclaredFieldsGenerator != null) {
                undeclaredFieldsGenerator.generate();
            }
            out.print(" }");
            if (printAfter != null) {
                out.print(printAfter);
            }
        }

    }

    private final File schemaFile;
    private final File outputDir;
    private Map<String, IAType> typeMap;
    private Map<String, TypeDataGen> typeAnnotMap;
    private DataGeneratorContext dgCtx;

    public AdmDataGen(File schemaFile, File outputDir) {
        this.schemaFile = schemaFile;
        this.outputDir = outputDir;
    }

    public void init() throws IOException, ParseException, AlgebricksException, ACIDException, MetadataException {
        FileReader aql = new FileReader(schemaFile);
        AQLParser parser = new AQLParser(aql);
        Query q = (Query) parser.Statement();
        aql.close();
        // TODO: Need to fix how to use transactions here.
        MetadataTransactionContext mdTxnCtx = new MetadataTransactionContext(-1);
        ADGenDmlTranslator dmlt = new ADGenDmlTranslator(mdTxnCtx, q.getPrologDeclList());
        dmlt.translate();
        AqlCompiledMetadataDeclarations acmd = dmlt.getCompiledDeclarations();
        typeMap = acmd.getTypeDeclarations();
        typeAnnotMap = acmd.getTypeDataGenMap();
        dgCtx = new DataGeneratorContext();
    }

    public void dataGen() throws Exception {
        for (Map.Entry<String, IAType> me : typeMap.entrySet()) {
            String tn = me.getKey();
            TypeDataGen tdg = typeAnnotMap.get(tn);
            if (tdg.isDataGen()) {
                IAType t = me.getValue();

                if (t.getTypeTag() != ATypeTag.RECORD) {
                    throw new NotImplementedException();
                }
                ARecordType rt = (ARecordType) t;
                RecordDataGenAnnotation dga = firstDataGenAnnotation(rt);
                if (dga == null) {
                    throw new Exception("No data generator annotations for type " + tn);
                }
                File outFile = new File(outputDir + File.separator + tdg.getOutputFileName());
                PrintStream outStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outFile)));
                RecordGenerator rg = new RecordGenerator(rt, dga, "\n");
                rg.init(outStream, dgCtx);
                for (long i = 0; i < tdg.getNumValues(); i++) {
                    rg.generate();
                }
                outStream.close();
            }
        }
    }

    private static RecordDataGenAnnotation firstDataGenAnnotation(ARecordType rt) {
        for (IRecordTypeAnnotation annot : rt.getAnnotations()) {
            if (annot.getKind() == Kind.RECORD_DATA_GEN) {
                return (RecordDataGenAnnotation) annot;
            }
        }
        return null;
    }

    private static String[] readFileAsStringArray(File file) throws IOException {
        List<String> tmp = new ArrayList<String>();
        FileInputStream fstream = new FileInputStream(file);
        DataInputStream in = new DataInputStream(fstream);
        BufferedReader br = new BufferedReader(new InputStreamReader(in));
        String strLine;
        while ((strLine = br.readLine()) != null) {
            tmp.add(strLine);
        }
        in.close();
        return tmp.toArray(new String[0]);
    }

    private static String getConstructor(IAType t) throws Exception {
        if (t instanceof BuiltinType) {
            String s = ((BuiltinType) t).getConstructor();
            if (s == null) {
                throw new Exception("Type " + t + " has no constructors.");
            }
            return s;
        } else {
            throw new Exception("No string constructor for type " + t);
        }
    }

}