tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject [33/48] git commit: TAJO-813: CLI should support comment character with multi-line query. (Hyoungjun Kim via hyunsik)
Date Tue, 20 May 2014 18:46:31 GMT
TAJO-813: CLI should support comment character with multi-line query. (Hyoungjun Kim via hyunsik)


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/8e523c12
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/8e523c12
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/8e523c12

Branch: refs/heads/window_function
Commit: 8e523c12e50ff660b09b4d6fa289ea617e08fbfa
Parents: eb4e54a
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Thu May 8 20:15:53 2014 +0900
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Thu May 8 20:15:53 2014 +0900

----------------------------------------------------------------------
 CHANGES                                         |   3 +
 .../java/org/apache/tajo/cli/ParsedResult.java  |  10 +-
 .../java/org/apache/tajo/cli/SimpleParser.java  | 179 ++++++++++++++++---
 .../main/java/org/apache/tajo/cli/TajoCli.java  |   6 +-
 .../org/apache/tajo/jdbc/TajoResultSetBase.java |   2 +-
 .../java/org/apache/tajo/QueryTestCaseBase.java |   8 +-
 .../org/apache/tajo/cli/TestSimpleParser.java   | 120 +++++++++++--
 .../apache/tajo/engine/eval/ExprTestBase.java   |   4 +-
 8 files changed, 278 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index 3c5237e..0620893 100644
--- a/CHANGES
+++ b/CHANGES
@@ -31,6 +31,9 @@ Release 0.9.0 - unreleased
 
   BUG FIXES
 
+    TAJO-813: CLI should support comment character with multi-line query.
+    (Hyoungjun Kim via hyunsik)
+
     TAJO-800: CLI's meta command should be aware "TABLE_NAME" style. 
     (Hyoungjun Kim via hyunsik)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java b/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java
index fb89678..001aded 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/ParsedResult.java
@@ -26,22 +26,28 @@ public class ParsedResult {
   }
 
   private final StatementType type;
+  private final String historyStatement;
   private final String statement;
 
-  public ParsedResult(StatementType type, String statement) {
+  public ParsedResult(StatementType type, String statement, String historyStatement) {
     this.type = type;
     this.statement = statement;
+    this.historyStatement = historyStatement;
   }
 
   public StatementType getType() {
     return type;
   }
 
+  public String getHistoryStatement() {
+    return historyStatement.trim();
+  }
+
   public String getStatement() {
     return statement.trim();
   }
 
   public String toString() {
-    return "(" + type.name() + ") " + statement;
+    return "(" + type.name() + ") " + historyStatement;
   }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java b/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java
index 61f6d2a..afb8a59 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/SimpleParser.java
@@ -36,7 +36,6 @@ public class SimpleParser {
     META,          // Meta Command
     STATEMENT,     // Statement
     WITHIN_QUOTE,  // Within Quote
-    COMMENT,
     INVALID,       // Invalid Statement
     STATEMENT_EOS, // End State (End of Statement)
     META_EOS       // End State (End of Statement)
@@ -44,7 +43,16 @@ public class SimpleParser {
 
   ParsingState state = START_STATE;
   int lineNum;
-  StringBuilder appender = new StringBuilder();
+
+  /**
+   * It will be used to store a query statement into Jline history.
+   * the query statement for history does not include unnecessary white spaces and new line.
+   */
+  private StringBuilder historyAppender = new StringBuilder();
+  /**
+   * It will be used to submit a query statement to the TajoMaster. It just contains a raw
query statement string.
+   */
+  private StringBuilder rawAppender = new StringBuilder();
 
   public static final ParsingState START_STATE = ParsingState.TOK_START;
 
@@ -80,6 +88,12 @@ public class SimpleParser {
     int idx = 0;
     char [] chars = str.toCharArray();
 
+    // if parsing continues, it means that the previous line is broken by '\n'.
+    // So, we should add new line to rawAppender.
+    if (isStatementContinue()) {
+      rawAppender.append("\n");
+    }
+
     while(idx < str.length()) {
 
       // initialization for new statement
@@ -107,30 +121,31 @@ public class SimpleParser {
         ////////////////////////////
         while (state != ParsingState.META_EOS && idx < chars.length) {
           char character = chars[idx++];
-          if (Character.isWhitespace(character)) {
-            // skip
-          } else if (isEndOfMeta(character)) {
+
+          if (isEndOfMeta(character)) {
             state = ParsingState.META_EOS;
+          } else if (Character.isWhitespace(character)) {
+            // skip
           }
         }
 
         if (state == ParsingState.META_EOS) {
-          appender.append(str.subSequence(lineStartIdx, idx - 1).toString());
+          historyAppender.append(str.subSequence(lineStartIdx, idx - 1).toString());
+          appendToRawStatement(str.subSequence(lineStartIdx, idx - 1).toString(), true);
         } else {
-          appender.append(str.subSequence(lineStartIdx, idx).toString());
+          historyAppender.append(str.subSequence(lineStartIdx, idx).toString());
+          appendToRawStatement(str.subSequence(lineStartIdx, idx).toString(), true);
         }
 
-      } else if (isCommentStart(chars[idx])) {
-        idx++;
-        while (!isLineEnd(chars[idx]) && idx < chars.length) {
-          idx++;
-        }
+      } else if (isInlineCommentStart(chars, idx)) {
+        idx = consumeInlineComment(chars, idx);
+        appendToRawStatement(str.subSequence(lineStartIdx, idx).toString(), true);
+
       /////////////////////////////////
       //    TOK_START     -> STATEMENT
       // or TOK_STATEMENT -> STATEMENT
       ////////////////////////////////
       } else if (isStatementContinue() || isStatementStart(chars[idx])) {
-        int endIdx = 0;
         if (!isStatementContinue()) { // TOK_START -> STATEMENT
           state = ParsingState.STATEMENT;
         }
@@ -138,9 +153,16 @@ public class SimpleParser {
         while (!isTerminateState(state) && idx < chars.length) {
           char character = chars[idx++];
 
+          ///////////////////////////////////////////////////////
+          // in-statement loop BEGIN
+          ///////////////////////////////////////////////////////
           if (isEndOfStatement(character)) {
             state = ParsingState.STATEMENT_EOS;
-            endIdx = idx - 1;
+
+          } else if (state == ParsingState.STATEMENT && character == '\n') {
+            appendToBothStatements(chars, lineStartIdx, idx, 1); // omit new line chacter
'\n' from history statement
+            lineStartIdx = idx;
+
           } else if (state == ParsingState.STATEMENT && character == '\'') { // TOK_STATEMENT
-> WITHIN_QUOTE
             state = ParsingState.WITHIN_QUOTE;
 
@@ -149,7 +171,21 @@ public class SimpleParser {
             } else {
               continue;
             }
+
+
+            // idx points the characters followed by the current character. So, we should
use 'idx - 1'
+            // in order to point the current character.
+          } else if (state == ParsingState.STATEMENT && idx < chars.length &&
isInlineCommentStart(chars, idx - 1)) {
+            idx++;
+            appendToBothStatements(chars, lineStartIdx, idx, 2); // omit two dash characters
'--' from history statement
+            int commentStartIdx = idx;
+            idx = consumeInlineComment(chars, idx);
+            appendToRawStatement(str.subSequence(commentStartIdx, idx).toString(), true);
+            lineStartIdx = idx;
           }
+          ///////////////////////////////////////////////////////
+          // in-statement loop END
+          ///////////////////////////////////////////////////////
 
           if (state == ParsingState.WITHIN_QUOTE) {
             while(idx < chars.length) {
@@ -168,14 +204,17 @@ public class SimpleParser {
           }
         }
 
-        if (state == ParsingState.STATEMENT_EOS) {
-          appender.append(str.subSequence(lineStartIdx, endIdx).toString());
+        // After all characters are consumed
+
+        if (state == ParsingState.STATEMENT_EOS) { // If one query statement is terminated
+          appendToBothStatements(chars, lineStartIdx, idx - 1); // skip semicolon (;)
         } else {
-          appender.append(str.subSequence(lineStartIdx, idx).toString());
+          appendToBothStatements(chars, lineStartIdx, idx);
 
-          // if it is not within quote and there is no space between lines, add a space.
-          if (state == ParsingState.STATEMENT && (appender.charAt(appender.length()
- 1) != ' ')) {
-            appender.append(" ");
+          // if it is not within quote and there is no space between lines, adds a space.
+          if (state == ParsingState.STATEMENT && (historyAppender.charAt(historyAppender.length()
- 1) != ' ')) {
+            historyAppender.append(" ");
+            rawAppender.append("\n");
           }
         }
       } else { // skip unknown character
@@ -189,6 +228,65 @@ public class SimpleParser {
     return statements;
   }
 
+  /**
+   * Append the range of characters into a given StringBuilder instance.
+   *
+   * @param chars Characters
+   * @param fromIdx start character index
+   * @param toIdx end character index
+   */
+  private void appendToStatement(StringBuilder builder, char[] chars, int fromIdx, int toIdx)
{
+    builder.append(chars, fromIdx, toIdx - fromIdx);
+  }
+
+  /**
+   * Append the range of characters into both history and raw appenders. It omits the number
of characters specified by
+   * <code>omitCharNums</code>.
+   *
+   *
+   * @param chars Characters
+   * @param fromIdx start character index
+   * @param toIdx end character index
+   * @param omitCharNums how many characters will be omitted from history statement
+   */
+  private void appendToBothStatements(char[] chars, int fromIdx, int toIdx, int omitCharNums)
{
+    appendToStatement(historyAppender, chars, fromIdx, toIdx - omitCharNums);
+    if (historyAppender.charAt(historyAppender.length() - 1) != ' ') {
+      historyAppender.append(" ");
+    }
+    appendToStatement(rawAppender, chars, fromIdx, toIdx);
+  }
+
+  /**
+   * Append the range of characters into both history and raw appenders.
+   *
+   *
+   * @param chars Characters
+   * @param fromIdx start character index
+   * @param toIdx end character index
+   */
+  private void appendToBothStatements(char[] chars, int fromIdx, int toIdx) {
+    historyAppender.append(chars, fromIdx, toIdx - fromIdx);
+    rawAppender.append(chars, fromIdx, toIdx - fromIdx);
+  }
+
+  private int consumeInlineComment(char [] chars, int currentIdx) {
+    currentIdx++;
+    while (currentIdx < chars.length && !isNewLine(chars[currentIdx])) {
+      currentIdx++;
+    }
+    return currentIdx;
+  }
+
+  private void appendToRawStatement(String str, boolean addLF) {
+    if (!str.isEmpty() && !"\n".equals(str) &&
+        rawAppender.length() > 0 && addLF && rawAppender.charAt(rawAppender.length()
- 1) != '\n') {
+      rawAppender.append(str);
+    } else {
+      rawAppender.append(str);
+    }
+  }
+
   private static boolean isEndOfMeta(char character) {
     return character == ';' || character == '\n';
   }
@@ -197,11 +295,21 @@ public class SimpleParser {
     return character == ';';
   }
 
-  private boolean isCommentStart(char character) {
-    return state == ParsingState.TOK_START && character == '-';
+  /**
+   * It checks if inline comment '--' begins.
+   * @param chars
+   * @param idx
+   * @return
+   */
+  private boolean isInlineCommentStart(char[] chars, int idx) {
+    if (idx >= chars.length - 1) {
+      return false;
+    }
+    return (state == ParsingState.STATEMENT || state == ParsingState.TOK_START) &&
+        (chars[idx] == '-' && chars[idx + 1] == '-');
   }
 
-  private boolean isLineEnd(char character) {
+  private boolean isNewLine(char character) {
     return character == '\n';
   }
 
@@ -213,6 +321,13 @@ public class SimpleParser {
     return state == ParsingState.WITHIN_QUOTE || state == ParsingState.STATEMENT;
   }
 
+  /**
+   * process all parsed statements so far and return a list of parsed results.
+   *
+   * @param endOfFile TRUE if the end of file.
+   * @return the list of parsed results, each of result contains one query statement or meta
command.
+   * @throws InvalidStatementException
+   */
   private List<ParsedResult> doProcessEndOfStatement(boolean endOfFile) throws InvalidStatementException
{
     List<ParsedResult> parsedResults = new ArrayList<ParsedResult>();
     String errorMessage = "";
@@ -228,24 +343,32 @@ public class SimpleParser {
     }
 
     if (isTerminateState(state)) {
-      String statement = appender.toString();
+      String historyStatement = historyAppender.toString();
+      String rawStatement = rawAppender.toString();
       if (state == ParsingState.META_EOS) {
-        parsedResults.add(new ParsedResult(META, statement));
+        parsedResults.add(new ParsedResult(META, rawStatement, historyStatement));
         state = ParsingState.TOK_START;
       } else if (state == ParsingState.STATEMENT_EOS) {
-        parsedResults.add(new ParsedResult(STATEMENT, statement));
+        parsedResults.add(new ParsedResult(STATEMENT, rawStatement, historyStatement));
       } else {
         throw new InvalidStatementException("ERROR: " + errorMessage);
       }
 
       // reset all states
-      appender.delete(0, appender.length());
+      historyAppender.delete(0, historyAppender.length());
+      rawAppender.delete(0, rawAppender.length());
       state = START_STATE;
     }
 
     return parsedResults;
   }
 
+  /**
+   * It manually triggers the end of file.
+   *
+   * @return the list of parsed results, each of result contains one query statement or meta
command.
+   * @throws InvalidStatementException
+   */
   public List<ParsedResult> EOF() throws InvalidStatementException {
     return doProcessEndOfStatement(true);
   }
@@ -259,6 +382,6 @@ public class SimpleParser {
   }
 
   public String toString() {
-    return "[" + state.name() + "]: " + appender.toString();
+    return "[" + state.name() + "]: " + historyAppender.toString();
   }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
index 2f9d5cf..7489351 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/TajoCli.java
@@ -305,7 +305,7 @@ public class TajoCli {
 
       if (parsedResults.size() > 0) {
         for (ParsedResult parsed : parsedResults) {
-          history.addStatement(parsed.getStatement() + (parsed.getType() == STATEMENT ? ";":""));
+          history.addStatement(parsed.getHistoryStatement() + (parsed.getType() == STATEMENT
? ";":""));
         }
       }
       executeParsedResults(parsedResults);
@@ -471,11 +471,11 @@ public class TajoCli {
 
   private void printUsage() {
     HelpFormatter formatter = new HelpFormatter();
-    formatter.printHelp( "tsql [options] [database]", options );
+    formatter.printHelp("tsql [options] [database]", options);
   }
 
   private void printInvalidCommand(String command) {
-    sout.println("Invalid command " + command +". Try \\? for help.");
+    sout.println("Invalid command " + command + ". Try \\? for help.");
   }
 
   public void close() {

http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java
index b3c53e1..4c307b3 100644
--- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java
+++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetBase.java
@@ -493,7 +493,7 @@ public abstract class TajoResultSetBase implements ResultSet {
 
   @Override
   public Statement getStatement() throws SQLException {
-    throw new SQLFeatureNotSupportedException("getStatement not supported");
+    throw new SQLFeatureNotSupportedException("getHistoryStatement not supported");
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index bf5891a..3843c58 100644
--- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -253,7 +253,7 @@ public class QueryTestCaseBase {
     if (parsedResults.size() > 1) {
       assertNotNull("This script \"" + queryFileName + "\" includes two or more queries");
     }
-    ResultSet result = client.executeQueryAndGetResult(parsedResults.get(0).getStatement());
+    ResultSet result = client.executeQueryAndGetResult(parsedResults.get(0).getHistoryStatement());
     assertNotNull("Query succeeded test", result);
     return result;
   }
@@ -470,13 +470,13 @@ public class QueryTestCaseBase {
 
     for (ParsedResult parsedResult : parsedResults) {
       // parse a statement
-      Expr expr = sqlParser.parse(parsedResult.getStatement());
+      Expr expr = sqlParser.parse(parsedResult.getHistoryStatement());
       assertNotNull(ddlFilePath + " cannot be parsed", expr);
 
       if (expr.getType() == OpType.CreateTable) {
         CreateTable createTable = (CreateTable) expr;
         String tableName = createTable.getTableName();
-        assertTrue("Table [" + tableName + "] creation is failed.", client.updateQuery(parsedResult.getStatement()));
+        assertTrue("Table [" + tableName + "] creation is failed.", client.updateQuery(parsedResult.getHistoryStatement()));
 
         TableDesc createdTable = client.getTableDesc(tableName);
         String createdTableName = createdTable.getName();
@@ -491,7 +491,7 @@ public class QueryTestCaseBase {
         String tableName = dropTable.getTableName();
         assertTrue("table '" + tableName + "' existence check",
             client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
-        assertTrue("table drop is failed.", client.updateQuery(parsedResult.getStatement()));
+        assertTrue("table drop is failed.", client.updateQuery(parsedResult.getHistoryStatement()));
         assertFalse("table '" + tableName + "' dropped check",
             client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName)));
         if (isLocalTable) {

http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java b/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java
index 9c02b65..5b5057b 100644
--- a/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java
+++ b/tajo-core/src/test/java/org/apache/tajo/cli/TestSimpleParser.java
@@ -47,68 +47,151 @@ public class TestSimpleParser {
     List<ParsedResult> res1 = SimpleParser.parseScript("\\d");
     assertEquals(1, res1.size());
     assertEquals(ParsedResult.StatementType.META, res1.get(0).getType());
-    assertEquals("\\d", res1.get(0).getStatement());
+    assertEquals("\\d", res1.get(0).getHistoryStatement());
 
     List<ParsedResult> res2 = SimpleParser.parseScript("\\d;\\c;\\f;");
     assertEquals(3, res2.size());
     assertEquals(ParsedResult.StatementType.META, res2.get(0).getType());
-    assertEquals("\\d", res2.get(0).getStatement());
+    assertEquals("\\d", res2.get(0).getHistoryStatement());
     assertEquals(ParsedResult.StatementType.META, res2.get(1).getType());
-    assertEquals("\\c", res2.get(1).getStatement());
+    assertEquals("\\c", res2.get(1).getHistoryStatement());
     assertEquals(ParsedResult.StatementType.META, res2.get(2).getType());
-    assertEquals("\\f", res2.get(2).getStatement());
+    assertEquals("\\f", res2.get(2).getHistoryStatement());
 
     List<ParsedResult> res3 = SimpleParser.parseScript("\n\t\t  \\d;\n\\c;\t\t\\f 
;");
     assertEquals(3, res3.size());
     assertEquals(ParsedResult.StatementType.META, res3.get(0).getType());
-    assertEquals("\\d", res3.get(0).getStatement());
+    assertEquals("\\d", res3.get(0).getHistoryStatement());
     assertEquals(ParsedResult.StatementType.META, res3.get(1).getType());
-    assertEquals("\\c", res3.get(1).getStatement());
+    assertEquals("\\c", res3.get(1).getHistoryStatement());
     assertEquals(ParsedResult.StatementType.META, res3.get(2).getType());
-    assertEquals("\\f", res3.get(2).getStatement());
+    assertEquals("\\f", res3.get(2).getHistoryStatement());
 
     List<ParsedResult> res4 = SimpleParser.parseScript("\\\td;");
     assertEquals(1, res4.size());
-    assertEquals("\\\td", res4.get(0).getStatement());
+    assertEquals("\\\td", res4.get(0).getHistoryStatement());
   }
 
   @Test
-  public final void testStatements() throws InvalidStatementException {
+  public final void testParseScript() throws InvalidStatementException {
     List<ParsedResult> res1 = SimpleParser.parseScript("select * from test;");
     assertEquals(1, res1.size());
     assertEquals(ParsedResult.StatementType.STATEMENT, res1.get(0).getType());
     assertEquals("select * from test", res1.get(0).getStatement());
+    assertEquals("select * from test", res1.get(0).getHistoryStatement());
 
     List<ParsedResult> res2 = SimpleParser.parseScript("select * from test;");
     assertEquals(1, res2.size());
     assertEquals(ParsedResult.StatementType.STATEMENT, res2.get(0).getType());
     assertEquals("select * from test", res2.get(0).getStatement());
+    assertEquals("select * from test", res2.get(0).getHistoryStatement());
 
     List<ParsedResult> res3 = SimpleParser.parseScript("select * from test1;select
* from test2;");
     assertEquals(2, res3.size());
     assertEquals(ParsedResult.StatementType.STATEMENT, res3.get(0).getType());
     assertEquals("select * from test1", res3.get(0).getStatement());
+    assertEquals("select * from test1", res3.get(0).getHistoryStatement());
     assertEquals(ParsedResult.StatementType.STATEMENT, res3.get(1).getType());
     assertEquals("select * from test2", res3.get(1).getStatement());
+    assertEquals("select * from test2", res3.get(1).getHistoryStatement());
 
     List<ParsedResult> res4 = SimpleParser.parseScript("\t\t\n\rselect * from \ntest1;select
* from test2\n;");
     assertEquals(2, res4.size());
     assertEquals(ParsedResult.StatementType.STATEMENT, res4.get(0).getType());
     assertEquals("select * from \ntest1", res4.get(0).getStatement());
+    assertEquals("select * from test1", res4.get(0).getHistoryStatement());
     assertEquals(ParsedResult.StatementType.STATEMENT, res4.get(1).getType());
     assertEquals("select * from test2", res4.get(1).getStatement());
+    assertEquals("select * from test2", res4.get(1).getHistoryStatement());
 
     List<ParsedResult> res5 =
         SimpleParser.parseScript("\t\t\n\rselect * from \ntest1;\\d test;select * from test2;\n\nselect
1;");
     assertEquals(4, res5.size());
     assertEquals(ParsedResult.StatementType.STATEMENT, res5.get(0).getType());
     assertEquals("select * from \ntest1", res5.get(0).getStatement());
+    assertEquals("select * from test1", res5.get(0).getHistoryStatement());
     assertEquals(ParsedResult.StatementType.META, res5.get(1).getType());
     assertEquals("\\d test", res5.get(1).getStatement());
     assertEquals(ParsedResult.StatementType.STATEMENT, res5.get(2).getType());
     assertEquals("select * from test2", res5.get(2).getStatement());
+    assertEquals("select * from test2", res5.get(2).getHistoryStatement());
     assertEquals(ParsedResult.StatementType.STATEMENT, res5.get(3).getType());
     assertEquals("select 1", res5.get(3).getStatement());
+    assertEquals("select 1", res5.get(3).getHistoryStatement());
+
+    List<ParsedResult> res6 =
+        SimpleParser.parseScript("select * from \n--test1; select * from test2;\ntest3;");
+    assertEquals(1, res6.size());
+    assertEquals("select * from test3", res6.get(0).getHistoryStatement());
+    assertEquals("select * from \n--test1; select * from test2;\ntest3", res6.get(0).getStatement());
+
+    List<ParsedResult> res7 =
+        SimpleParser.parseScript("select * from --test1; select * from test2;\ntest3;");
+    assertEquals(1, res7.size());
+    assertEquals("select * from test3", res7.get(0).getHistoryStatement());
+    assertEquals("select * from --test1; select * from test2;\ntest3", res7.get(0).getStatement());
+
+    List<ParsedResult> res8 = SimpleParser.parseScript("\\d test\nselect * \n--from
test1;\nfrom test2;\\d test2;");
+    assertEquals(3, res8.size());
+    assertEquals(ParsedResult.StatementType.META, res8.get(0).getType());
+    assertEquals("\\d test", res8.get(0).getStatement());
+    assertEquals("\\d test", res8.get(0).getHistoryStatement());
+    assertEquals(ParsedResult.StatementType.STATEMENT, res8.get(1).getType());
+    assertEquals("select * \n--from test1;\nfrom test2", res8.get(1).getStatement());
+    assertEquals("select * from test2", res8.get(1).getHistoryStatement());
+    assertEquals(ParsedResult.StatementType.META, res8.get(2).getType());
+    assertEquals("\\d test2", res8.get(2).getStatement());
+    assertEquals("\\d test2", res8.get(2).getHistoryStatement());
+  }
+
+  @Test
+  public final void testParseLines() throws InvalidStatementException {
+    SimpleParser simpleParser = new SimpleParser();
+    List<ParsedResult> res1 = null;
+
+    res1 = simpleParser.parseLines("select * from test1; select * from test2;");
+    assertEquals(2, res1.size());
+    assertEquals("select * from test1", res1.get(0).getStatement());
+    assertEquals("select * from test2", res1.get(1).getStatement());
+    assertEquals("select * from test1", res1.get(0).getHistoryStatement());
+    assertEquals("select * from test2", res1.get(1).getHistoryStatement());
+
+    simpleParser = new SimpleParser();
+    res1 = simpleParser.parseLines("select * from ");
+    assertEquals(0, res1.size());
+    res1 = simpleParser.parseLines("test1; select * from test2;");
+    assertEquals(2, res1.size());
+    assertEquals("select * from \ntest1", res1.get(0).getStatement());
+    assertEquals("select * from test2", res1.get(1).getStatement());
+    assertEquals("select * from test1", res1.get(0).getHistoryStatement());
+    assertEquals("select * from test2", res1.get(1).getHistoryStatement());
+
+    // select * from
+    // --test1; select * from test2;
+    // test3;
+    simpleParser = new SimpleParser();
+    res1 = simpleParser.parseLines("select * from ");
+    assertEquals(0, res1.size());
+    res1 = simpleParser.parseLines("--test1; select * from test2;");
+    assertEquals(0, res1.size());
+    res1 = simpleParser.parseLines("test3;");
+    assertEquals(1, res1.size());
+    assertEquals("select * from test3", res1.get(0).getHistoryStatement());
+    assertEquals("select * from \n--test1; select * from test2;\ntest3", res1.get(0).getStatement());
+
+
+    // select * from
+    // test1 --select * from test2;
+    // where col1 = '123';
+    simpleParser = new SimpleParser();
+    res1 = simpleParser.parseLines("select * from ");
+    assertEquals(0, res1.size());
+    res1 = simpleParser.parseLines("test1 --select * from test2;");
+    assertEquals(0, res1.size());
+    res1 = simpleParser.parseLines("where col1 = '123';");
+    assertEquals(1, res1.size());
+    assertEquals("select * from test1 where col1 = '123'", res1.get(0).getHistoryStatement());
+    assertEquals("select * from \ntest1 --select * from test2;\nwhere col1 = '123'", res1.get(0).getStatement());
   }
 
   @Test
@@ -116,13 +199,21 @@ public class TestSimpleParser {
     List<ParsedResult> res1 = SimpleParser.parseScript("select '\n;' from test;");
     assertEquals(1, res1.size());
     assertEquals(ParsedResult.StatementType.STATEMENT, res1.get(0).getType());
+    assertEquals("select '\n;' from test", res1.get(0).getHistoryStatement());
     assertEquals("select '\n;' from test", res1.get(0).getStatement());
 
     List<ParsedResult> res2 = SimpleParser.parseScript("select 'abc\nbbc\nddf' from
test;");
     assertEquals(1, res2.size());
     assertEquals(ParsedResult.StatementType.STATEMENT, res2.get(0).getType());
+    assertEquals("select 'abc\nbbc\nddf' from test", res2.get(0).getHistoryStatement());
     assertEquals("select 'abc\nbbc\nddf' from test", res2.get(0).getStatement());
 
+    List<ParsedResult> res3 = SimpleParser.parseScript("select '--test', \n'--test2'
from test");
+    assertEquals(1, res3.size());
+    assertEquals(ParsedResult.StatementType.STATEMENT, res3.get(0).getType());
+    assertEquals("select '--test', '--test2' from test", res3.get(0).getHistoryStatement());
+    assertEquals("select '--test', \n'--test2' from test", res3.get(0).getStatement());
+
     try {
       SimpleParser.parseScript("select 'abc");
       assertTrue(false);
@@ -144,7 +235,8 @@ public class TestSimpleParser {
     assertEquals(0, result2.size());
     List<ParsedResult> result3 = parser.EOF();
     assertEquals(1, result3.size());
-    assertEquals(lines[0] + lines[1], result3.get(0).getStatement());
+    assertEquals(lines[0] + lines[1], result3.get(0).getHistoryStatement());
+    assertEquals(lines[0] + "\n" + lines[1], result3.get(0).getStatement());
   }
 
   @Test
@@ -158,8 +250,8 @@ public class TestSimpleParser {
     assertEquals(0, result1.size());
     List<ParsedResult> result2 = parser.parseLines(lines[1]);
     assertEquals(2, result2.size());
-    assertEquals("select abc, 'bbc' from test", result2.get(0).getStatement());
-    assertEquals("select * from test3", result2.get(1).getStatement());
+    assertEquals("select abc, 'bbc' from test", result2.get(0).getHistoryStatement());
+    assertEquals("select * from test3", result2.get(1).getHistoryStatement());
   }
 
   @Test
@@ -173,7 +265,7 @@ public class TestSimpleParser {
     assertEquals(0, result1.size());
     List<ParsedResult> result2 = parser.parseLines(lines[1]);
     assertEquals(2, result2.size());
-    assertEquals("select abc, 'bbc' from test", result2.get(0).getStatement());
-    assertEquals("select * from test3", result2.get(1).getStatement());
+    assertEquals("select abc, 'bbc' from test", result2.get(0).getHistoryStatement());
+    assertEquals("select * from test3", result2.get(1).getHistoryStatement());
   }
 }

http://git-wip-us.apache.org/repos/asf/tajo/blob/8e523c12/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
index 979bee2..bab6ec7 100644
--- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java
@@ -105,7 +105,7 @@ public class ExprTestBase {
     if (parsedResults.size() > 1) {
       throw new RuntimeException("this query includes two or more statements.");
     }
-    Expr expr = analyzer.parse(parsedResults.get(0).getStatement());
+    Expr expr = analyzer.parse(parsedResults.get(0).getHistoryStatement());
     VerificationState state = new VerificationState();
     preLogicalPlanVerifier.verify(session, state, expr);
     if (state.getErrorMessages().size() > 0) {
@@ -124,7 +124,7 @@ public class ExprTestBase {
 
     Target [] targets = plan.getRootBlock().getRawTargets();
     if (targets == null) {
-      throw new PlanningException("Wrong query statement or query plan: " + parsedResults.get(0).getStatement());
+      throw new PlanningException("Wrong query statement or query plan: " + parsedResults.get(0).getHistoryStatement());
     }
     for (Target t : targets) {
       assertJsonSerDer(t.getEvalTree());


Mime
View raw message