Return-Path: X-Original-To: apmail-asterixdb-commits-archive@minotaur.apache.org Delivered-To: apmail-asterixdb-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4E07E19C31 for ; Sat, 26 Mar 2016 20:27:39 +0000 (UTC) Received: (qmail 4758 invoked by uid 500); 26 Mar 2016 20:27:39 -0000 Delivered-To: apmail-asterixdb-commits-archive@asterixdb.apache.org Received: (qmail 4719 invoked by uid 500); 26 Mar 2016 20:27:39 -0000 Mailing-List: contact commits-help@asterixdb.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.incubator.apache.org Delivered-To: mailing list commits@asterixdb.incubator.apache.org Received: (qmail 4710 invoked by uid 99); 26 Mar 2016 20:27:39 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 26 Mar 2016 20:27:39 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 909E8C1404 for ; Sat, 26 Mar 2016 20:27:38 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -4.216 X-Spam-Level: X-Spam-Status: No, score=-4.216 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.996] autolearn=disabled Received: from mx1-lw-us.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id cWgrT3IwiR3b for ; Sat, 26 Mar 2016 20:27:35 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-us.apache.org (ASF Mail Server at mx1-lw-us.apache.org) with SMTP id B25905FB0C for ; Sat, 26 Mar 2016 20:27:33 +0000 (UTC) Received: (qmail 4583 invoked by uid 99); 26 Mar 2016 20:27:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 26 Mar 2016 20:27:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 82025DFB73; Sat, 26 Mar 2016 20:27:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: amoudi@apache.org To: commits@asterixdb.incubator.apache.org Date: Sat, 26 Mar 2016 20:27:34 -0000 Message-Id: <85685115244b4be7a2dc88b3db36d92a@git.apache.org> In-Reply-To: <30a4853f56cf456ea6901b8d2197b3d5@git.apache.org> References: <30a4853f56cf456ea6901b8d2197b3d5@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [03/13] incubator-asterixdb git commit: Improve Error Handling in Local Directory Feeds http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java ---------------------------------------------------------------------- diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java index c9ce47c..75b2800 100644 --- a/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java +++ b/asterix-external-data/src/test/java/org/apache/asterix/external/classad/test/FunctionalTester.java @@ -38,6 +38,7 @@ import org.apache.asterix.external.classad.Lexer.TokenType; import org.apache.asterix.external.classad.PrettyPrint; import org.apache.asterix.external.classad.StringLexerSource; import org.apache.asterix.external.classad.Value; +import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool; import org.apache.asterix.external.library.ClassAdParser; import org.apache.asterix.om.base.AMutableString; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -54,15 +55,15 @@ public class FunctionalTester { public ExprTreeHolder tree; public Value value; - public Variable(String name, ExprTree tree) { + public Variable(String name, ExprTree tree, ClassAdObjectPool objectPool) { this.name = name; - this.tree = new ExprTreeHolder(tree); + this.tree = new ExprTreeHolder(tree, objectPool); this.isTree = true; } - public Variable() { + public Variable(ClassAdObjectPool objectPool) { this.name = null; - this.tree = new ExprTreeHolder(null); + this.tree = new ExprTreeHolder(objectPool); this.isTree = false; } @@ -73,8 +74,9 @@ public class FunctionalTester { this.tree = null; } - public void getStringRepresentation(AMutableCharArrayString representation) throws HyracksDataException { - ClassAdUnParser unparser = new PrettyPrint(); + public void getStringRepresentation(AMutableCharArrayString representation, ClassAdObjectPool objectPool) + throws HyracksDataException { + ClassAdUnParser unparser = new PrettyPrint(objectPool); if (isTree) { unparser.unparse(representation, tree); @@ -179,7 +181,7 @@ public class FunctionalTester { * *--------------------------------------------------------------------*/ - public static int test(int argc, String[] argv) throws IOException { + public static int test(int argc, String[] argv, ClassAdObjectPool objectPool) throws IOException { // here boolean quit; AMutableString line = new AMutableString(null); @@ -194,10 +196,10 @@ public class FunctionalTester { boolean good_line; Command command; - good_line = replace_variables(line, state, parameters); + good_line = replace_variables(line, state, parameters, objectPool); if (good_line) { command = get_command(line, parameters); - quit = handle_command(command, line, state, parameters); + quit = handle_command(command, line, state, parameters, objectPool); } } print_final_state(state); @@ -327,14 +329,14 @@ public class FunctionalTester { * * @throws HyracksDataException *********************************************************************/ - public static boolean replace_variables(AMutableString mutableLine, State state, Parameters parameters) - throws HyracksDataException { + public static boolean replace_variables(AMutableString mutableLine, State state, Parameters parameters, + ClassAdObjectPool objectPool) throws HyracksDataException { boolean good_line; String error; good_line = true; error = ""; - Variable var = new Variable(); + Variable var = new Variable(objectPool); for (;;) { int dollar; int current_position; @@ -366,7 +368,7 @@ public class FunctionalTester { error += "'"; break; } - var.getStringRepresentation(variable_value); + var.getStringRepresentation(variable_value, objectPool); // We have to be careful with substr() because with gcc 2.96, it likes to // assert/except if you give it values that are too large. @@ -457,8 +459,8 @@ public class FunctionalTester { * * @throws IOException *********************************************************************/ - public static boolean handle_command(Command command, AMutableString line, State state, Parameters parameters) - throws IOException { + public static boolean handle_command(Command command, AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) throws IOException { boolean quit = false; switch (command) { @@ -469,31 +471,31 @@ public class FunctionalTester { print_error_message("Unknown command on line", state); break; case cmd_Let: - handle_let(line, state, parameters); + handle_let(line, state, parameters, objectPool); break; case cmd_Eval: - handle_eval(line, state, parameters); + handle_eval(line, state, parameters, objectPool); break; case cmd_Print: - handle_print(line, state, parameters); + handle_print(line, state, parameters, objectPool); break; case cmd_Same: - handle_same(line, state, parameters); + handle_same(line, state, parameters, objectPool); break; case cmd_Sameq: - handle_sameq(line, state, parameters); + handle_sameq(line, state, parameters, objectPool); break; case cmd_Diff: - handle_diff(line, state, parameters); + handle_diff(line, state, parameters, objectPool); break; case cmd_Diffq: - handle_diffq(line, state, parameters); + handle_diffq(line, state, parameters, objectPool); break; case cmd_Set: - handle_set(line, state, parameters); + handle_set(line, state, parameters, objectPool); break; case cmd_Show: - handle_show(line, state, parameters); + handle_show(line, state, parameters, objectPool); break; case cmd_Writexml: // handle_writexml(line, state, parameters); @@ -520,18 +522,19 @@ public class FunctionalTester { * * @throws IOException *********************************************************************/ - public static void handle_let(AMutableString line, State state, Parameters parameters) throws IOException { + public static void handle_let(AMutableString line, State state, Parameters parameters, ClassAdObjectPool objectPool) + throws IOException { AMutableString variable_name = new AMutableString(null); ExprTree tree; Variable variable; if (get_variable_name(line, true, variable_name, state, parameters)) { - tree = get_expr(line, state, parameters); + tree = get_expr(line, state, parameters, objectPool); if (tree != null) { - variable = new Variable(variable_name.getStringValue(), tree); + variable = new Variable(variable_name.getStringValue(), tree, objectPool); variables.put(variable_name.getStringValue(), variable); if (parameters.interactive) { - print_expr(tree, state, parameters); + print_expr(tree, state, parameters, objectPool); } } } @@ -544,16 +547,17 @@ public class FunctionalTester { * * @throws IOException *********************************************************************/ - public static void handle_eval(AMutableString line, State state, Parameters parameters) throws IOException { + public static void handle_eval(AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) throws IOException { AMutableString variable_name = new AMutableString(""); ExprTree tree; Variable variable; if (get_variable_name(line, true, variable_name, state, parameters)) { - tree = get_expr(line, state, parameters); + tree = get_expr(line, state, parameters, objectPool); if (tree != null) { - Value value = new Value(); - if (!evaluate_expr(tree, value, parameters)) { + Value value = new Value(objectPool); + if (!evaluate_expr(tree, value, parameters, objectPool)) { print_error_message("Couldn't evaluate rvalue", state); } else { variable = new Variable(variable_name.getStringValue(), value); @@ -570,11 +574,12 @@ public class FunctionalTester { * * @throws IOException *********************************************************************/ - public static void handle_print(AMutableString line, State state, Parameters parameters) throws IOException { + public static void handle_print(AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) throws IOException { ExprTree tree; - tree = get_expr(line, state, parameters); + tree = get_expr(line, state, parameters, objectPool); if (tree != null) { - print_expr(tree, state, parameters); + print_expr(tree, state, parameters, objectPool); } } @@ -584,25 +589,26 @@ public class FunctionalTester { * * @throws IOException *********************************************************************/ - public static void handle_same(AMutableString line, State state, Parameters parameters) throws IOException { - ExprTreeHolder tree = new ExprTreeHolder(); - ExprTreeHolder tree2 = new ExprTreeHolder(); - Value value1 = new Value(); - Value value2 = new Value(); + public static void handle_same(AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) throws IOException { + ExprTreeHolder tree = new ExprTreeHolder(objectPool); + ExprTreeHolder tree2 = new ExprTreeHolder(objectPool); + Value value1 = new Value(objectPool); + Value value2 = new Value(objectPool); try { - get_two_exprs(line, tree, tree2, state, parameters); + get_two_exprs(line, tree, tree2, state, parameters, objectPool); if (tree.getInnerTree() != null || tree2.getInnerTree() != null) { if (parameters.debug) { System.out.println("Sameeval has two trees:"); System.out.print(" "); - print_expr(tree, state, parameters); + print_expr(tree, state, parameters, objectPool); System.out.print(" "); - print_expr(tree2, state, parameters); + print_expr(tree2, state, parameters, objectPool); } - if (!evaluate_expr(tree, value1, parameters)) { + if (!evaluate_expr(tree, value1, parameters, objectPool)) { print_error_message("Couldn't evaluate first expression.\n", state); - } else if (!evaluate_expr(tree2, value2, parameters)) { + } else if (!evaluate_expr(tree2, value2, parameters, objectPool)) { print_error_message("Couldn't evaluate second expressions.\n", state); } else if (!value1.sameAs(value2)) { print_error_message("the expressions are different.", state); @@ -624,14 +630,16 @@ public class FunctionalTester { /********************************************************************* * Function: handle_sameq * Purpose: + * @param objectPool * * @throws IOException *********************************************************************/ - public static void handle_sameq(AMutableString line, State state, Parameters parameters) throws IOException { - ExprTreeHolder tree = new ExprTreeHolder(); - ExprTreeHolder tree2 = new ExprTreeHolder(); + public static void handle_sameq(AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) throws IOException { + ExprTreeHolder tree = new ExprTreeHolder(objectPool); + ExprTreeHolder tree2 = new ExprTreeHolder(objectPool); - get_two_exprs(line, tree, tree2, state, parameters); + get_two_exprs(line, tree, tree2, state, parameters, objectPool); if (tree.getInnerTree() != null || tree2.getInnerTree() != null) { if (!tree.sameAs(tree2)) { print_error_message("the expressions are different.", state); @@ -646,17 +654,18 @@ public class FunctionalTester { * * @throws IOException *********************************************************************/ - public static void handle_diff(AMutableString line, State state, Parameters parameters) throws IOException { - ExprTreeHolder tree = new ExprTreeHolder(); - ExprTreeHolder tree2 = new ExprTreeHolder(); - Value value1 = new Value(); - Value value2 = new Value(); - - get_two_exprs(line, tree, tree2, state, parameters); + public static void handle_diff(AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) throws IOException { + ExprTreeHolder tree = new ExprTreeHolder(objectPool); + ExprTreeHolder tree2 = new ExprTreeHolder(objectPool); + Value value1 = new Value(objectPool); + Value value2 = new Value(objectPool); + + get_two_exprs(line, tree, tree2, state, parameters, objectPool); if (tree.getInnerTree() != null || tree2.getInnerTree() != null) { - if (!evaluate_expr(tree, value1, parameters)) { + if (!evaluate_expr(tree, value1, parameters, objectPool)) { print_error_message("Couldn't evaluate first expression.\n", state); - } else if (!evaluate_expr(tree2, value2, parameters)) { + } else if (!evaluate_expr(tree2, value2, parameters, objectPool)) { print_error_message("Couldn't evaluate second expressions.\n", state); } else if (value1.sameAs(value2)) { print_error_message("the expressions are the same.", state); @@ -669,14 +678,16 @@ public class FunctionalTester { /********************************************************************* * Function: handle_diffq * Purpose: + * @param objectPool * * @throws IOException *********************************************************************/ - public static void handle_diffq(AMutableString line, State state, Parameters parameters) throws IOException { - ExprTreeHolder tree = new ExprTreeHolder(); - ExprTreeHolder tree2 = new ExprTreeHolder(); + public static void handle_diffq(AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) throws IOException { + ExprTreeHolder tree = new ExprTreeHolder(objectPool); + ExprTreeHolder tree2 = new ExprTreeHolder(objectPool); - get_two_exprs(line, tree, tree2, state, parameters); + get_two_exprs(line, tree, tree2, state, parameters, objectPool); if (tree.getInnerTree() != null || tree2.getInnerTree() != null) { if (tree.sameAs(tree2)) { print_error_message("the expressions are the same.", state); @@ -688,8 +699,10 @@ public class FunctionalTester { /********************************************************************* * Function: handle_set * Purpose: + * @param objectPool *********************************************************************/ - public static void handle_set(AMutableString line, State state, Parameters parameters) { + public static void handle_set(AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) { AMutableString option_name = new AMutableString(null); AMutableString option_value = new AMutableString(null); @@ -718,8 +731,10 @@ public class FunctionalTester { /********************************************************************* * Function: handle_show * Purpose: + * @param objectPool *********************************************************************/ - public static void handle_show(AMutableString line, State state, Parameters parameters) { + public static void handle_show(AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) { AMutableString option_name = new AMutableString(null); if (get_variable_name(line, false, option_name, state, parameters)) { @@ -906,10 +921,11 @@ public class FunctionalTester { * * @throws IOException *********************************************************************/ - public static ExprTree get_expr(AMutableString line, State state, Parameters parameters) throws IOException { + public static ExprTree get_expr(AMutableString line, State state, Parameters parameters, + ClassAdObjectPool objectPool) throws IOException { int offset; ExprTree tree; - ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null); + ClassAdParser parser = new ClassAdParser(objectPool); StringLexerSource lexer_source = new StringLexerSource(line.getStringValue()); tree = parser.parseExpression(lexer_source, false); @@ -930,9 +946,9 @@ public class FunctionalTester { * @throws IOException *********************************************************************/ public static void get_two_exprs(AMutableString line, ExprTreeHolder tree1, ExprTreeHolder tree2, State state, - Parameters parameters) throws IOException { + Parameters parameters, ClassAdObjectPool objectPool) throws IOException { int offset; - ClassAdParser parser = new ClassAdParser(null, false, true, false, null, null, null); + ClassAdParser parser = new ClassAdParser(objectPool); StringLexerSource lexer_source = new StringLexerSource(line.getStringValue()); tree1.setInnerTree(parser.parseExpression(lexer_source, false)); @@ -943,7 +959,7 @@ public class FunctionalTester { } else { if (parameters.debug) { System.out.print("# Tree1: "); - print_expr(tree1, state, parameters); + print_expr(tree1, state, parameters, objectPool); } if (parser.peekToken() != TokenType.LEX_COMMA) { @@ -961,9 +977,9 @@ public class FunctionalTester { throw new IOException(); } else if (parameters.debug) { System.out.print("# Tree2: "); - print_expr(tree2, state, parameters); + print_expr(tree2, state, parameters, objectPool); System.out.print("# Tree1: "); - print_expr(tree1, state, parameters); + print_expr(tree1, state, parameters, objectPool); System.out.println(); } } @@ -978,14 +994,15 @@ public class FunctionalTester { * * @throws HyracksDataException *********************************************************************/ - public static void print_expr(ExprTree tree, State state, Parameters parameters) throws HyracksDataException { + public static void print_expr(ExprTree tree, State state, Parameters parameters, ClassAdObjectPool objectPool) + throws HyracksDataException { AMutableCharArrayString output = new AMutableCharArrayString(); if (state.format == PrintFormat.print_Compact) { - ClassAdUnParser unparser = new ClassAdUnParser(); + ClassAdUnParser unparser = new ClassAdUnParser(objectPool); unparser.unparse(output, tree); } else if (state.format == PrintFormat.print_Pretty) { - PrettyPrint unparser = new PrettyPrint(); + PrettyPrint unparser = new PrettyPrint(objectPool); unparser.unparse(output, tree); } else if (state.format == PrintFormat.print_XML) {/* * ClassAdXMLUnParser unparser = new @@ -1009,8 +1026,9 @@ public class FunctionalTester { * * @throws HyracksDataException *********************************************************************/ - public static boolean evaluate_expr(ExprTree tree, Value value, Parameters parameters) throws HyracksDataException { - ClassAd classad = new ClassAd(); + public static boolean evaluate_expr(ExprTree tree, Value value, Parameters parameters, ClassAdObjectPool objectPool) + throws HyracksDataException { + ClassAd classad = new ClassAd(objectPool); boolean success = false;; classad.insert("internal___", tree); success = classad.evaluateAttr("internal___", value); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java ---------------------------------------------------------------------- diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java index d6db6c2..5fd6f21 100644 --- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java +++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParser.java @@ -59,15 +59,7 @@ import org.apache.asterix.external.classad.StringLexerSource; import org.apache.asterix.external.classad.TokenValue; import org.apache.asterix.external.classad.Value; import org.apache.asterix.external.classad.Value.NumberFactor; -import org.apache.asterix.external.classad.object.pool.AttributeReferencePool; -import org.apache.asterix.external.classad.object.pool.BitSetPool; -import org.apache.asterix.external.classad.object.pool.ClassAdPool; -import org.apache.asterix.external.classad.object.pool.ExprHolderPool; -import org.apache.asterix.external.classad.object.pool.ExprListPool; -import org.apache.asterix.external.classad.object.pool.LiteralPool; -import org.apache.asterix.external.classad.object.pool.OperationPool; -import org.apache.asterix.external.classad.object.pool.TokenValuePool; -import org.apache.asterix.external.classad.object.pool.ValuePool; +import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool; import org.apache.asterix.external.parser.AbstractDataParser; import org.apache.asterix.om.base.ABoolean; import org.apache.asterix.om.base.AMutableInt32; @@ -94,16 +86,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars private Lexer lexer = new Lexer(); private LexerSource currentSource = null; private boolean isExpr = false; - // object pools - private final ExprHolderPool mutableExprPool = new ExprHolderPool(); - private final TokenValuePool tokenValuePool = new TokenValuePool(); - private final ClassAdPool classAdPool = new ClassAdPool(); - private final ExprListPool exprListPool = new ExprListPool(); - private final ValuePool valuePool = new ValuePool(); - private final LiteralPool literalPool = new LiteralPool(); - private final BitSetPool bitSetPool = new BitSetPool(); - private final OperationPool operationPool = new OperationPool(); - private final AttributeReferencePool attrRefPool = new AttributeReferencePool(); + private final ClassAdObjectPool objectPool; // asterix objects private ARecordType recordType; private IObjectPool recordBuilderPool = new ListObjectPool( @@ -112,17 +95,22 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars new ListBuilderFactory()); private IObjectPool abvsBuilderPool = new ListObjectPool( new AbvsBuilderFactory()); - private ClassAd rootAd = new ClassAd(false, true); + private final ClassAd rootAd; private String exprPrefix = "expr="; private String exprSuffix = ""; private boolean evaluateExpr = true; private String exprFieldNameSuffix = "Expr"; private boolean keepBoth = true; - private boolean oldFormat = false; + private boolean oldFormat = true; private StringLexerSource stringLexerSource = new StringLexerSource(""); public ClassAdParser(ARecordType recordType, boolean oldFormat, boolean evaluateExpr, boolean keepBoth, - String exprPrefix, String exprSuffix, String exprFieldNameSuffix) { + String exprPrefix, String exprSuffix, String exprFieldNameSuffix, ClassAdObjectPool objectPool) { + if (objectPool == null) { + System.out.println(); + } + this.objectPool = objectPool; + this.rootAd = new ClassAd(objectPool); this.recordType = recordType; this.currentSource = new CharArrayLexerSource(); this.recordType = recordType; @@ -137,6 +125,15 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars this.exprFieldNameSuffix = exprFieldNameSuffix; } + public ClassAdParser(ClassAdObjectPool objectPool) { + if (objectPool == null) { + System.out.println(); + } + this.objectPool = objectPool; + this.currentSource = new CharArrayLexerSource(); + rootAd = null; + } + /*********************************** * AsterixDB Specific begin * @@ -160,8 +157,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars return false; } tt = lexer.peekToken(); - TokenValue tv = tokenValuePool.get(); - ExprTreeHolder tree = mutableExprPool.get(); + TokenValue tv = objectPool.tokenValuePool.get(); + ExprTreeHolder tree = objectPool.mutableExprPool.get(); while (tt != TokenType.LEX_CLOSE_BOX) { // Get the name of the expression tv.reset(); @@ -200,8 +197,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // we will store a string representation of the expression int len = lexer.getLexSource().getPosition() - positionBefore; // add it as it is to the classAd - Literal lit = literalPool.get(); - Value exprVal = valuePool.get(); + Literal lit = objectPool.literalPool.get(); + Value exprVal = objectPool.valuePool.get(); exprVal.setStringValue(exprPrefix + String.valueOf(lexer.getLexSource().getBuffer(), positionBefore, len) + exprSuffix); Literal.createLiteral(lit, exprVal, NumberFactor.NO_FACTOR); @@ -263,15 +260,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars listBuilderPool.reset(); recordBuilderPool.reset(); abvsBuilderPool.reset(); - mutableExprPool.reset(); - tokenValuePool.reset(); - classAdPool.reset(); - exprListPool.reset(); - valuePool.reset(); - literalPool.reset(); - bitSetPool.reset(); - operationPool.reset(); - attrRefPool.reset(); + objectPool.reset(); } private ATypeTag getTargetTypeTag(ATypeTag expectedTypeTag, IAType aObjectType) throws IOException { @@ -384,7 +373,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars case EXPR_LIST_NODE: case FN_CALL_NODE: case OP_NODE: - val = valuePool.get(); + val = objectPool.valuePool.get(); if (pAd.evaluateAttr(name, val)) { } else { @@ -393,7 +382,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars } break; case LITERAL_NODE: - val = ((Literal) tree).getValue(); + val = ((Literal) tree.getTree()).getValue(); break; default: throw new HyracksDataException("Unknown Expression type detected: " + tree.getKind()); @@ -574,7 +563,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars } private BitSet getBitSet() { - return bitSetPool.get(); + return objectPool.bitSetPool.get(); } public static int checkNullConstraints(ARecordType recType, BitSet nulls) { @@ -710,7 +699,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars public ClassAd parseClassAd(LexerSource lexer_source, boolean full) throws IOException { System.out.println("Don't use this call. instead, pass a mutable classad instance"); - ClassAd ad = classAdPool.get(); + ClassAd ad = objectPool.classAdPool.get(); if (lexer.initialize(lexer_source)) { if (!parseClassAd(ad, full)) { return null; @@ -804,7 +793,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars */ public ExprTree parseExpression(String buffer, boolean full) throws IOException { stringLexerSource.setNewSource(buffer); - ExprTreeHolder mutableExpr = mutableExprPool.get(); + ExprTreeHolder mutableExpr = objectPool.mutableExprPool.get(); if (lexer.initialize(stringLexerSource)) { parseExpression(mutableExpr, full); } @@ -816,7 +805,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars } public ExprTree parseExpression(LexerSource lexer_source, boolean full) throws IOException { - ExprTreeHolder mutableExpr = mutableExprPool.get(); + ExprTreeHolder mutableExpr = objectPool.mutableExprPool.get(); if (lexer.initialize(lexer_source)) { parseExpression(mutableExpr, full); } @@ -827,7 +816,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars if (!lexer.wasInitialized()) { return null; } else { - ExprTreeHolder expr = mutableExprPool.get(); + ExprTreeHolder expr = objectPool.mutableExprPool.get(); parseExpression(expr, false); ExprTree innerTree = expr.getInnerTree(); return innerTree; @@ -849,20 +838,21 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars private boolean parseExpression(ExprTreeHolder tree, boolean full) throws IOException { TokenType tt; - if (!parseLogicalORExpression(tree)) + if (!parseLogicalORExpression(tree)) { return false; + } if ((tt = lexer.peekToken()) == TokenType.LEX_QMARK) { lexer.consumeToken(); ExprTreeHolder treeL = tree; - ExprTreeHolder treeM = mutableExprPool.get(); - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeM = objectPool.mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); parseExpression(treeM); if ((tt = lexer.consumeToken()) != TokenType.LEX_COLON) { throw new HyracksDataException("expected LEX_COLON, but got " + Lexer.strLexToken(tt)); } parseExpression(treeR); if (treeL.getInnerTree() != null && treeM.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(Operation.OpKind_TERNARY_OP, treeL, treeM, treeR, newTree); tree.setInnerTree(newTree); return (true); @@ -882,15 +872,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // | LogicalORExpression '||' LogicalANDExpression private boolean parseLogicalORExpression(ExprTreeHolder tree) throws IOException { - if (!parseLogicalANDExpression(tree)) + if (!parseLogicalANDExpression(tree)) { return false; + } while ((lexer.peekToken()) == TokenType.LEX_LOGICAL_OR) { ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); lexer.consumeToken(); parseLogicalANDExpression(treeR); if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(Operation.OpKind_LOGICAL_OR_OP, treeL, treeR, null, newTree); tree.setInnerTree(newTree); } else { @@ -904,15 +895,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // LogicalANDExpression .= InclusiveORExpression // | LogicalANDExpression '&&' InclusiveORExpression private boolean parseLogicalANDExpression(ExprTreeHolder tree) throws IOException { - if (!parseInclusiveORExpression(tree)) + if (!parseInclusiveORExpression(tree)) { return false; + } while ((lexer.peekToken()) == TokenType.LEX_LOGICAL_AND) { ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); lexer.consumeToken(); parseInclusiveORExpression(treeR); if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(Operation.OpKind_LOGICAL_AND_OP, treeL, treeR, null, newTree); tree.setInnerTree(newTree); } else { @@ -926,15 +918,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // InclusiveORExpression .= ExclusiveORExpression // | InclusiveORExpression '|' ExclusiveORExpression public boolean parseInclusiveORExpression(ExprTreeHolder tree) throws IOException { - if (!parseExclusiveORExpression(tree)) + if (!parseExclusiveORExpression(tree)) { return false; + } while ((lexer.peekToken()) == TokenType.LEX_BITWISE_OR) { ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); lexer.consumeToken(); parseExclusiveORExpression(treeR); if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(Operation.OpKind_BITWISE_OR_OP, treeL, treeR, null, newTree); tree.setInnerTree(newTree); } else { @@ -948,15 +941,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // ExclusiveORExpression .= ANDExpression // | ExclusiveORExpression '^' ANDExpression private boolean parseExclusiveORExpression(ExprTreeHolder tree) throws IOException { - if (!parseANDExpression(tree)) + if (!parseANDExpression(tree)) { return false; + } while ((lexer.peekToken()) == TokenType.LEX_BITWISE_XOR) { lexer.consumeToken(); ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); parseANDExpression(treeR); if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(Operation.OpKind_BITWISE_XOR_OP, treeL, treeR, null, newTree); tree.setInnerTree(newTree); } else { @@ -970,15 +964,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // ANDExpression .= EqualityExpression // | ANDExpression '&' EqualityExpression private boolean parseANDExpression(ExprTreeHolder tree) throws IOException { - if (!parseEqualityExpression(tree)) + if (!parseEqualityExpression(tree)) { return false; + } while ((lexer.peekToken()) == TokenType.LEX_BITWISE_AND) { ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); lexer.consumeToken(); parseEqualityExpression(treeR); if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(Operation.OpKind_BITWISE_AND_OP, treeL, treeR, null, newTree); tree.setInnerTree(newTree); } else { @@ -997,13 +992,14 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars private boolean parseEqualityExpression(ExprTreeHolder tree) throws IOException { TokenType tt; int op = Operation.OpKind_NO_OP; - if (!parseRelationalExpression(tree)) + if (!parseRelationalExpression(tree)) { return false; + } tt = lexer.peekToken(); while (tt == TokenType.LEX_EQUAL || tt == TokenType.LEX_NOT_EQUAL || tt == TokenType.LEX_META_EQUAL || tt == TokenType.LEX_META_NOT_EQUAL) { ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); lexer.consumeToken(); parseRelationalExpression(treeR); switch (tt) { @@ -1023,7 +1019,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars throw new HyracksDataException("ClassAd: Should not reach here"); } if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(op, treeL, treeR, null, newTree); tree.setInnerTree(newTree); } else { @@ -1042,14 +1038,15 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // | RelationalExpression '>=' ShiftExpression private boolean parseRelationalExpression(ExprTreeHolder tree) throws IOException { TokenType tt; - if (!parseShiftExpression(tree)) + if (!parseShiftExpression(tree)) { return false; + } tt = lexer.peekToken(); while (tt == TokenType.LEX_LESS_THAN || tt == TokenType.LEX_GREATER_THAN || tt == TokenType.LEX_LESS_OR_EQUAL || tt == TokenType.LEX_GREATER_OR_EQUAL) { int op = Operation.OpKind_NO_OP; ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); lexer.consumeToken(); parseShiftExpression(treeR); switch (tt) { @@ -1069,7 +1066,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars throw new HyracksDataException("ClassAd: Should not reach here"); } if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(op, treeL, treeR, null, newTree); tree.setInnerTree(newTree); } else { @@ -1086,13 +1083,14 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // | ShiftExpression '>>' AdditiveExpression // | ShiftExpression '>>>' AditiveExpression private boolean parseShiftExpression(ExprTreeHolder tree) throws IOException { - if (!parseAdditiveExpression(tree)) + if (!parseAdditiveExpression(tree)) { return false; + } TokenType tt = lexer.peekToken(); while (tt == TokenType.LEX_LEFT_SHIFT || tt == TokenType.LEX_RIGHT_SHIFT || tt == TokenType.LEX_URIGHT_SHIFT) { ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); int op; lexer.consumeToken(); parseAdditiveExpression(treeR); @@ -1112,7 +1110,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars } if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(op, treeL, treeR, null, newTree); tree.setInnerTree(newTree); } else { @@ -1128,17 +1126,18 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // | AdditiveExpression '+' MultiplicativeExpression // | AdditiveExpression '-' MultiplicativeExpression private boolean parseAdditiveExpression(ExprTreeHolder tree) throws IOException { - if (!parseMultiplicativeExpression(tree)) + if (!parseMultiplicativeExpression(tree)) { return false; + } TokenType tt = lexer.peekToken(); while (tt == TokenType.LEX_PLUS || tt == TokenType.LEX_MINUS) { ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); lexer.consumeToken(); parseMultiplicativeExpression(treeR); if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation( (tt == TokenType.LEX_PLUS) ? Operation.OpKind_ADDITION_OP : Operation.OpKind_SUBTRACTION_OP, treeL, treeR, null, newTree); @@ -1157,13 +1156,14 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // | MultiplicativeExpression '/' UnaryExpression // | MultiplicativeExpression '%' UnaryExpression private boolean parseMultiplicativeExpression(ExprTreeHolder tree) throws IOException { - if (!parseUnaryExpression(tree)) + if (!parseUnaryExpression(tree)) { return false; + } TokenType tt = lexer.peekToken(); while (tt == TokenType.LEX_MULTIPLY || tt == TokenType.LEX_DIVIDE || tt == TokenType.LEX_MODULUS) { ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); int op; lexer.consumeToken(); parseUnaryExpression(treeR); @@ -1182,7 +1182,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars throw new HyracksDataException("ClassAd: Should not reach here"); } if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(op, treeL, treeR, null, newTree); tree.setInnerTree(newTree); } else { @@ -1202,7 +1202,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars if (tt == TokenType.LEX_MINUS || tt == TokenType.LEX_PLUS || tt == TokenType.LEX_BITWISE_NOT || tt == TokenType.LEX_LOGICAL_NOT) { lexer.consumeToken(); - ExprTreeHolder treeM = mutableExprPool.get(); + ExprTreeHolder treeM = objectPool.mutableExprPool.get(); int op = Operation.OpKind_NO_OP; parseUnaryExpression(treeM); switch (tt) { @@ -1222,7 +1222,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars throw new HyracksDataException("ClassAd: Shouldn't Get here"); } if (treeM.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(op, treeM, null, null, newTree); tree.setInnerTree(newTree); } else { @@ -1240,18 +1240,19 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // | PostfixExpression '[' Expression ']' private boolean parsePostfixExpression(ExprTreeHolder tree) throws IOException { TokenType tt; - if (!parsePrimaryExpression(tree)) + if (!parsePrimaryExpression(tree)) { return false; + } while ((tt = lexer.peekToken()) == TokenType.LEX_OPEN_BOX || tt == TokenType.LEX_SELECTION) { ExprTreeHolder treeL = tree; - ExprTreeHolder treeR = mutableExprPool.get(); - TokenValue tv = tokenValuePool.get(); + ExprTreeHolder treeR = objectPool.mutableExprPool.get(); + TokenValue tv = objectPool.tokenValuePool.get(); lexer.consumeToken(); if (tt == TokenType.LEX_OPEN_BOX) { // subscript operation parseExpression(treeR); if (treeL.getInnerTree() != null && treeR.getInnerTree() != null) { - Operation newTree = operationPool.get(); + Operation newTree = objectPool.operationPool.get(); Operation.createOperation(Operation.OpKind_SUBSCRIPT_OP, treeL, treeR, null, newTree); if (lexer.consumeToken() == TokenType.LEX_CLOSE_BOX) { tree.setInnerTree(newTree); @@ -1266,7 +1267,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars throw new HyracksDataException("second argument of selector must be an " + "identifier (got" + String.valueOf(Lexer.strLexToken(tt)) + ")"); } - AttributeReference newTree = attrRefPool.get(); + AttributeReference newTree = objectPool.attrRefPool.get(); AttributeReference.createAttributeReference(treeL, tv.getStrValue(), false, newTree); tree.setInnerTree(newTree); } @@ -1285,7 +1286,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // ( ArgumentList non-terminal includes parentheses ) private boolean parsePrimaryExpression(ExprTreeHolder tree) throws IOException { ExprTreeHolder treeL; - TokenValue tv = tokenValuePool.get(); + TokenValue tv = objectPool.tokenValuePool.get(); TokenType tt; tree.setInnerTree(null); switch ((tt = lexer.peekToken(tv))) { @@ -1295,22 +1296,24 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars lexer.consumeToken(); // check for funcion call if ((tt = lexer.peekToken()) == TokenType.LEX_OPEN_PAREN) { - ExprList argList = exprListPool.get(); + ExprList argList = objectPool.exprListPool.get(); if (!parseArgumentList(argList)) { tree.setInnerTree(null); return false; - }; + } ; // special case function-calls should be converted // into a literal expression if the argument is a // string literal if (shouldEvaluateAtParseTime(tv.getStrValue().toString(), argList)) { tree.setInnerTree(evaluateFunction(tv.getStrValue().toString(), argList)); } else { - tree.setInnerTree(FunctionCall.createFunctionCall(tv.getStrValue().toString(), argList)); + tree.setInnerTree( + FunctionCall.createFunctionCall(tv.getStrValue().toString(), argList, objectPool)); } } else { // I don't think this is ever hit - tree.setInnerTree(AttributeReference.createAttributeReference(null, tv.getStrValue(), false)); + tree.setInnerTree( + AttributeReference.createAttributeReference(null, tv.getStrValue(), false, objectPool)); } return (tree.getInnerTree() != null); case LEX_SELECTION: @@ -1318,7 +1321,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars lexer.consumeToken(); if ((tt = lexer.consumeToken(tv)) == TokenType.LEX_IDENTIFIER) { // the boolean final arg signifies that reference is absolute - tree.setInnerTree(AttributeReference.createAttributeReference(null, tv.getStrValue(), true)); + tree.setInnerTree( + AttributeReference.createAttributeReference(null, tv.getStrValue(), true, objectPool)); return (tree.size() != 0); } // not an identifier following the '.' @@ -1328,7 +1332,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars case LEX_OPEN_PAREN: { isExpr = true; lexer.consumeToken(); - treeL = mutableExprPool.get(); + treeL = objectPool.mutableExprPool.get(); parseExpression(treeL); if (treeL.getInnerTree() == null) { tree.resetExprTree(null); @@ -1341,13 +1345,13 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // return false; } // assume make operation will return a new tree - tree.setInnerTree(Operation.createOperation(Operation.OpKind_PARENTHESES_OP, treeL)); + tree.setInnerTree(Operation.createOperation(Operation.OpKind_PARENTHESES_OP, treeL, objectPool)); return (tree.size() != 0); } - // constants + // constants case LEX_OPEN_BOX: { isExpr = true; - ClassAd newAd = classAdPool.get(); + ClassAd newAd = objectPool.classAdPool.get(); if (!parseClassAd(newAd)) { tree.resetExprTree(null); return false; @@ -1358,7 +1362,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars case LEX_OPEN_BRACE: { isExpr = true; - ExprList newList = exprListPool.get(); + ExprList newList = objectPool.exprListPool.get(); if (!parseExprList(newList)) { tree.setInnerTree(null); return false; @@ -1368,66 +1372,66 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars return true; case LEX_UNDEFINED_VALUE: { - Value val = valuePool.get(); + Value val = objectPool.valuePool.get(); lexer.consumeToken(); val.setUndefinedValue(); - tree.setInnerTree(Literal.createLiteral(val)); + tree.setInnerTree(Literal.createLiteral(val, objectPool)); return (tree.getInnerTree() != null); } case LEX_ERROR_VALUE: { - Value val = valuePool.get(); + Value val = objectPool.valuePool.get(); lexer.consumeToken(); val.setErrorValue(); - tree.setInnerTree(Literal.createLiteral(val)); + tree.setInnerTree(Literal.createLiteral(val, objectPool)); return (tree.getInnerTree() != null); } case LEX_BOOLEAN_VALUE: { - Value val = valuePool.get(); + Value val = objectPool.valuePool.get(); MutableBoolean b = new MutableBoolean(); tv.getBoolValue(b); lexer.consumeToken(); val.setBooleanValue(b); - tree.setInnerTree(Literal.createLiteral(val)); + tree.setInnerTree(Literal.createLiteral(val, objectPool)); return (tree.getInnerTree() != null); } case LEX_INTEGER_VALUE: { - Value val = valuePool.get(); + Value val = objectPool.valuePool.get(); lexer.consumeToken(); val.setIntegerValue(tv.getIntValue()); - tree.setInnerTree(Literal.createLiteral(val, tv.getFactor())); + tree.setInnerTree(Literal.createLiteral(val, tv.getFactor(), objectPool)); return (tree.getInnerTree() != null); } case LEX_REAL_VALUE: { - Value val = valuePool.get(); + Value val = objectPool.valuePool.get(); lexer.consumeToken(); val.setRealValue(tv.getRealValue()); - tree.setInnerTree(Literal.createLiteral(val, tv.getFactor())); + tree.setInnerTree(Literal.createLiteral(val, tv.getFactor(), objectPool)); return (tree.getInnerTree() != null); } case LEX_STRING_VALUE: { - Value val = valuePool.get(); + Value val = objectPool.valuePool.get(); lexer.consumeToken(); val.setStringValue(tv.getStrValue()); - tree.setInnerTree(Literal.createLiteral(val)); + tree.setInnerTree(Literal.createLiteral(val, objectPool)); return (tree.getInnerTree() != null); } case LEX_ABSOLUTE_TIME_VALUE: { - Value val = valuePool.get(); + Value val = objectPool.valuePool.get(); lexer.consumeToken(); val.setAbsoluteTimeValue(tv.getTimeValue()); - tree.setInnerTree(Literal.createLiteral(val)); + tree.setInnerTree(Literal.createLiteral(val, objectPool)); return (tree.getInnerTree() != null); } case LEX_RELATIVE_TIME_VALUE: { - Value val = valuePool.get(); + Value val = objectPool.valuePool.get(); lexer.consumeToken(); val.setRelativeTimeValue(tv.getTimeValue().getRelativeTime()); - tree.setInnerTree(Literal.createLiteral(val)); + tree.setInnerTree(Literal.createLiteral(val, objectPool)); return (tree.getInnerTree() != null); } @@ -1448,7 +1452,7 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // return false; } tt = lexer.peekToken(); - ExprTreeHolder tree = mutableExprPool.get(); + ExprTreeHolder tree = objectPool.mutableExprPool.get(); while (tt != TokenType.LEX_CLOSE_PAREN) { // parse the expression tree.reset(); @@ -1462,9 +1466,9 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // the next token must be a ',' or a ')' // or it can be a ';' if using old ClassAd semantics tt = lexer.peekToken(); - if (tt == TokenType.LEX_COMMA || (tt == TokenType.LEX_SEMICOLON && false)) + if (tt == TokenType.LEX_COMMA || (tt == TokenType.LEX_SEMICOLON && false)) { lexer.consumeToken(); - else if (tt != TokenType.LEX_CLOSE_PAREN) { + } else if (tt != TokenType.LEX_CLOSE_PAREN) { argList.clear(); throw new HyracksDataException( "expected LEX_COMMA or LEX_CLOSE_PAREN but got " + String.valueOf(Lexer.strLexToken(tt))); @@ -1490,11 +1494,12 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars public boolean parseClassAd(ClassAd ad, boolean full) throws IOException { TokenType tt; ad.clear(); - if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BOX) + if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BOX) { return false; + } tt = lexer.peekToken(); - TokenValue tv = tokenValuePool.get(); - ExprTreeHolder tree = mutableExprPool.get(); + TokenValue tv = objectPool.tokenValuePool.get(); + ExprTreeHolder tree = objectPool.mutableExprPool.get(); while (tt != TokenType.LEX_CLOSE_BOX) { // Get the name of the expression tv.reset(); @@ -1567,8 +1572,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars public boolean parseExprList(ExprList list, boolean full) throws IOException { TokenType tt; - ExprTreeHolder tree = new ExprTreeHolder(); - ExprList loe = new ExprList(); + ExprTreeHolder tree = objectPool.mutableExprPool.get(); + ExprList loe = objectPool.exprListPool.get(); if ((tt = lexer.consumeToken()) != TokenType.LEX_OPEN_BRACE) { throw new HyracksDataException( @@ -1589,16 +1594,16 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars // the next token must be a ',' or a '}' tt = lexer.peekToken(); - if (tt == TokenType.LEX_COMMA) + if (tt == TokenType.LEX_COMMA) { lexer.consumeToken(); - else if (tt != TokenType.LEX_CLOSE_BRACE) { + } else if (tt != TokenType.LEX_CLOSE_BRACE) { throw new HyracksDataException("while parsing expression list: expected " + "LEX_CLOSE_BRACE or LEX_COMMA but got " + Lexer.strLexToken(tt)); } } lexer.consumeToken(); - list.setValue(ExprList.createExprList(loe)); + list.setValue(ExprList.createExprList(loe, objectPool)); // if a full parse was requested, ensure that input is exhausted if (full && (lexer.consumeToken() != TokenType.LEX_END_OF_INPUT)) { @@ -1613,8 +1618,8 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars boolean should_eval = false; if (functionName.equalsIgnoreCase("absTime") || functionName.equalsIgnoreCase("relTime")) { if (argList.size() == 1 && argList.get(0).getKind() == NodeKind.LITERAL_NODE) { - Value val = new Value(); - AMutableNumberFactor factor = new AMutableNumberFactor(); + Value val = objectPool.valuePool.get(); + AMutableNumberFactor factor = objectPool.numFactorPool.get(); ((Literal) argList.get(0)).getComponents(val, factor); if (val.isStringValue()) { should_eval = true; @@ -1625,22 +1630,22 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars } public ExprTree evaluateFunction(String functionName, ExprList argList) throws HyracksDataException { - Value val = new Value(); - AMutableNumberFactor factor = new AMutableNumberFactor(); - ExprTreeHolder tree = new ExprTreeHolder(); + Value val = objectPool.valuePool.get(); + AMutableNumberFactor factor = objectPool.numFactorPool.get(); + ExprTreeHolder tree = objectPool.mutableExprPool.get(); ((Literal) argList.get(0)).getComponents(val, factor); - AMutableCharArrayString string_value = new AMutableCharArrayString(); + AMutableCharArrayString string_value = objectPool.strPool.get(); if (val.isStringValue(string_value)) { if (functionName.equalsIgnoreCase("absTime")) { - tree.setInnerTree(Literal.createAbsTime(string_value)); + tree.setInnerTree(Literal.createAbsTime(string_value, objectPool)); } else if (functionName.equalsIgnoreCase("relTime")) { - tree.setInnerTree(Literal.createRelTime(string_value)); + tree.setInnerTree(Literal.createRelTime(string_value, objectPool)); } else { - tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList)); + tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList, objectPool)); } } else { - tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList)); + tree.setInnerTree(FunctionCall.createFunctionCall(functionName, argList, objectPool)); } return tree; } @@ -1688,17 +1693,10 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars this.currentSource = lexerSource; } - public void reset() { - resetPools(); - } - - public Literal getLiteral() { - return literalPool.get(); - } - @Override public void parse(IRawRecord record, DataOutput out) throws IOException { try { + resetPools(); if (oldFormat) { int maxOffset = record.size(); rootAd.clear(); @@ -1718,7 +1716,6 @@ public class ClassAdParser extends AbstractDataParser implements IRecordDataPars line = readLine(buffer, aInt32, maxOffset); } } else { - resetPools(); currentSource.setNewSource(record.get()); rootAd.reset(); asterixParseClassAd(rootAd); http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java ---------------------------------------------------------------------- diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java index 5064992..17b83c2 100644 --- a/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java +++ b/asterix-external-data/src/test/java/org/apache/asterix/external/library/ClassAdParserFactory.java @@ -25,6 +25,7 @@ import java.util.Map; import org.apache.asterix.external.api.IRecordDataParser; import org.apache.asterix.external.api.IRecordDataParserFactory; +import org.apache.asterix.external.classad.object.pool.ClassAdObjectPool; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.om.types.ARecordType; import org.apache.hyracks.api.context.IHyracksTaskContext; @@ -42,8 +43,8 @@ public class ClassAdParserFactory implements IRecordDataParserFactory { private ARecordType recordType; private Map configuration; private boolean oldFormat = false; - private boolean evaluateExpr = true; - private boolean keepBoth; + private boolean evaluateExpr = false; + private boolean keepBoth = false; private String exprPrefix; private String exprSuffix; private String exprFieldNameSuffix; @@ -77,14 +78,12 @@ public class ClassAdParserFactory implements IRecordDataParserFactory { } parserConfig = configuration.get(ClassAdParserFactory.KEY_EVALUATE); - if ((parserConfig != null) && parserConfig.equalsIgnoreCase("false")) { - evaluateExpr = false; - keepBoth = false; + if ((parserConfig != null) && parserConfig.equalsIgnoreCase(ExternalDataConstants.TRUE)) { + evaluateExpr = true; } parserConfig = configuration.get(ClassAdParserFactory.KEY_KEEP_EXPR); - if ((parserConfig != null) && parserConfig.equalsIgnoreCase("false")) { - keepBoth = false; - evaluateExpr = true; + if ((parserConfig != null) && parserConfig.equalsIgnoreCase(ExternalDataConstants.TRUE)) { + keepBoth = true; } parserConfig = configuration.get(ClassAdParserFactory.KEY_EXPR_PREFIX); @@ -110,7 +109,7 @@ public class ClassAdParserFactory implements IRecordDataParserFactory { @Override public IRecordDataParser createRecordParser(IHyracksTaskContext ctx) throws HyracksDataException { return new ClassAdParser(recordType, oldFormat, evaluateExpr, keepBoth, exprPrefix, exprSuffix, - exprFieldNameSuffix); + exprFieldNameSuffix, new ClassAdObjectPool()); } @Override http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java ---------------------------------------------------------------------- diff --git a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java b/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java index 521cee0..851a7e0 100644 --- a/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java +++ b/asterix-external-data/src/test/java/org/apache/asterix/external/parser/test/RecordWithMetaTest.java @@ -28,7 +28,7 @@ import java.util.List; import org.apache.asterix.external.api.IRawRecord; import org.apache.asterix.external.input.record.converter.CSVToRecordWithMetadataAndPKConverter; import org.apache.asterix.external.input.record.reader.stream.QuotedLineRecordReader; -import org.apache.asterix.external.input.stream.LocalFileSystemInputStream; +import org.apache.asterix.external.input.stream.LocalFSInputStream; import org.apache.asterix.external.parser.ADMDataParser; import org.apache.asterix.external.parser.RecordWithMetadataParser; import org.apache.asterix.external.util.ExternalDataConstants; @@ -42,7 +42,9 @@ import org.apache.hyracks.algebricks.data.IPrinter; import org.apache.hyracks.algebricks.data.IPrinterFactory; import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; +import org.apache.hyracks.dataflow.std.file.FileSplit; import org.junit.Assert; import org.junit.Test; @@ -78,8 +80,10 @@ public class RecordWithMetaTest { int[] pkIndicators = { 1 }; // create input stream - LocalFileSystemInputStream inputStream = new LocalFileSystemInputStream( - Paths.get(getClass().getResource("/beer.csv").toURI()).toAbsolutePath(), null, false); + LocalFSInputStream inputStream = new LocalFSInputStream( + new FileSplit[] { new FileSplit(null, + new FileReference(Paths.get(getClass().getResource("/beer.csv").toURI()).toFile())) }, + null, null, 0, null, false); // create reader record reader QuotedLineRecordReader lineReader = new QuotedLineRecordReader(true, inputStream, null, http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/121e1d9a/asterix-external-data/src/test/resources/functional_tests.txt ---------------------------------------------------------------------- diff --git a/asterix-external-data/src/test/resources/functional_tests.txt b/asterix-external-data/src/test/resources/functional_tests.txt index 42c0e95..3aaae46 100644 --- a/asterix-external-data/src/test/resources/functional_tests.txt +++ b/asterix-external-data/src/test/resources/functional_tests.txt @@ -1,3 +1,14 @@ +echo Testing select on lists... +eval x = { + [a = 3; b = "I see London"], + [a = 2; b = "I see France"], + [a = 1; b = "I see Alain's funky pants"] + } +same $x.a, {3, 2, 1} +same $x.b, {"I see London", "I see France", "I see Alain's funky pants"} +same $x.c, {undefined, undefined, undefined} +same {}.a, {} + ///////////////////////////////// echo Testing basic math... eval x = (1 + 2) @@ -74,16 +85,7 @@ same isundefined($x.e), false // same $y, $z[1] ///////////////////////////////// -echo Testing select on lists... -eval x = { - [a = 3; b = "I see London"], - [a = 2; b = "I see France"], - [a = 1; b = "I see Alain's funky pants"] - } -same $x.a, {3, 2, 1} -same $x.b, {"I see London", "I see France", "I see Alain's funky pants"} -same $x.c, {undefined, undefined, undefined} -same {}.a, {} + ///////////////////////////////// echo Testing subscripts