tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jihoon...@apache.org
Subject [5/5] tajo git commit: Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support
Date Mon, 29 Jun 2015 06:46:52 GMT
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into index_support

Conflicts:
	CHANGES
	tajo-common/src/main/java/org/apache/tajo/SessionVars.java
	tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/806469a2
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/806469a2
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/806469a2

Branch: refs/heads/index_support
Commit: 806469a26f7228b0ce7b6f0134dd72ee109accc3
Parents: 17dfe86 773c3eb
Author: Jihoon Son <jihoonson@apache.org>
Authored: Mon Jun 29 15:29:01 2015 +0900
Committer: Jihoon Son <jihoonson@apache.org>
Committed: Mon Jun 29 15:46:39 2015 +0900

----------------------------------------------------------------------
 CHANGES                                         |  14 +-
 .../org/apache/tajo/catalog/CatalogUtil.java    |   4 +-
 .../java/org/apache/tajo/catalog/TableMeta.java |  92 +-------
 .../main/java/org/apache/tajo/SessionVars.java  |   5 +
 .../java/org/apache/tajo/conf/TajoConf.java     |   7 +-
 .../tajo/engine/function/builtin/Corr.java      | 224 +++++++++++++++++++
 .../planner/physical/BSTIndexScanExec.java      |  10 -
 .../planner/physical/EmptyScanIterator.java     |  40 ++++
 .../engine/planner/physical/SeqScanExec.java    | 111 +++++----
 .../tajo/engine/utils/TupleCacheScanner.java    | 114 ----------
 .../ws/rs/resources/QueryResultResource.java    |  35 +--
 tajo-core/src/main/proto/InternalTypes.proto    |   9 +
 .../engine/function/TestBuiltinFunctions.java   |  33 +++
 .../tajo/engine/query/TestOuterJoinQuery.java   |   2 +-
 .../tajo/engine/query/TestTablePartitions.java  | 106 ++++-----
 .../rs/resources/TestQueryResultResource.java   |  73 +-----
 .../TestTajoCli/testHelpSessionVars.result      |   1 +
 .../tajo/plan/LogicalPlanPreprocessor.java      |   5 +-
 .../org/apache/tajo/plan/LogicalPlanner.java    |  10 +-
 .../org/apache/tajo/plan/TablePropertyUtil.java |  92 ++++++++
 .../org/apache/tajo/plan/util/PlannerUtil.java  |  28 ---
 .../apache/tajo/storage/AbstractScanner.java    |  27 ++-
 .../org/apache/tajo/storage/MergeScanner.java   |   5 +-
 .../org/apache/tajo/storage/NullScanner.java    |   8 +-
 .../java/org/apache/tajo/storage/Scanner.java   |   9 +-
 .../apache/tajo/storage/SeekableScanner.java    |   4 +-
 .../apache/tajo/storage/hbase/HBaseScanner.java |   6 +-
 .../java/org/apache/tajo/storage/CSVFile.java   |   4 +-
 .../org/apache/tajo/storage/FileScanner.java    |   6 -
 .../org/apache/tajo/storage/FileTablespace.java |   7 +-
 .../java/org/apache/tajo/storage/RawFile.java   |   7 +
 .../java/org/apache/tajo/storage/RowFile.java   |   7 +
 .../apache/tajo/storage/avro/AvroScanner.java   |   7 +
 .../tajo/storage/parquet/ParquetScanner.java    |   7 +
 .../org/apache/tajo/storage/rcfile/RCFile.java  |   7 +
 .../sequencefile/SequenceFileScanner.java       |   7 +
 .../tajo/storage/text/DelimitedTextFile.java    |   7 +-
 .../apache/tajo/storage/TestMergeScanner.java   |   2 +-
 .../org/apache/tajo/storage/TestStorages.java   |  10 +-
 39 files changed, 667 insertions(+), 485 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/CHANGES
----------------------------------------------------------------------
diff --cc CHANGES
index ffed05c,90ef54a..746ed92
--- a/CHANGES
+++ b/CHANGES
@@@ -161,10 -166,9 +166,13 @@@ Release 0.11.0 - unrelease
  
    BUG FIXES
  
 +    TAJO-1608: Fix test failure in index_support branch. (jihoon)
 +
 +    TAJO-1594: Catalog schema is invalid for some databases. (jihoon)
 +
+     TAJO-1644: When inserting empty data into a partitioned table,
+     existing data would be removed. (jaehwa)
+ 
      TAJO-1642: CatalogServer need to check meta table first. (jaehwa)
  
      TAJO-1650: TestQueryResource.testGetAllQueries() occasionally fails.

http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
----------------------------------------------------------------------
diff --cc tajo-common/src/main/java/org/apache/tajo/SessionVars.java
index ba6a0cd,28fdb0b..9fa14c2
--- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
+++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java
@@@ -126,10 -126,10 +126,15 @@@ public enum SessionVars implements Conf
    NULL_CHAR(ConfVars.$TEXT_NULL, "null char of text file output", DEFAULT),
    CODEGEN(ConfVars.$CODEGEN, "Runtime code generation enabled (experiment)", DEFAULT),
  
 +  // for index
 +  INDEX_ENABLED(ConfVars.$INDEX_ENABLED, "index scan enabled", DEFAULT),
 +  INDEX_SELECTIVITY_THRESHOLD(ConfVars.$INDEX_SELECTIVITY_THRESHOLD, "the selectivity threshold
for index scan", DEFAULT),
 +
++  // for partition overwrite
+   PARTITION_NO_RESULT_OVERWRITE_ENABLED(ConfVars.$PARTITION_NO_RESULT_OVERWRITE_ENABLED,
+     "If True, a partitioned table is overwritten even if a sub query leads to no result.
"
+     + "Otherwise, the table data will be kept if there is no result", DEFAULT),
+ 
    // Behavior Control ---------------------------------------------------------
    ARITHABORT(ConfVars.$BEHAVIOR_ARITHMETIC_ABORT,
        "If true, a running query will be terminated when an overflow or divide-by-zero occurs.",
DEFAULT),

http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
index 521b6b9,54abca8..19da87b
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java
@@@ -96,98 -67,12 +96,88 @@@ public class BSTIndexScanExec extends P
      this.reader.open();
    }
  
 +  private static Schema mergeSubSchemas(Schema originalSchema, Schema subSchema, Target[]
targets, EvalNode qual) {
 +    Schema mergedSchema = new Schema();
 +    Set<Column> qualAndTargets = TUtil.newHashSet();
 +    qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(qual));
 +    for (Target target : targets) {
 +      qualAndTargets.addAll(EvalTreeUtil.findUniqueColumns(target.getEvalTree()));
 +    }
 +    for (Column column : originalSchema.getRootColumns()) {
 +      if (subSchema.contains(column)
 +          || qualAndTargets.contains(column)
 +          || qualAndTargets.contains(column)) {
 +        mergedSchema.addColumn(column);
 +      }
 +    }
 +    return mergedSchema;
 +  }
 +
    @Override
    public void init() throws IOException {
 +    Schema projected;
 +
 +    // in the case where projected column or expression are given
 +    // the target can be an empty list.
 +    if (plan.hasTargets()) {
 +      projected = new Schema();
 +      Set<Column> columnSet = new HashSet<Column>();
 +
 +      if (plan.hasQual()) {
 +        columnSet.addAll(EvalTreeUtil.findUniqueColumns(qual));
 +      }
 +
 +      for (Target t : plan.getTargets()) {
 +        columnSet.addAll(EvalTreeUtil.findUniqueColumns(t.getEvalTree()));
 +      }
 +
 +      for (Column column : inSchema.getAllColumns()) {
 +        if (columnSet.contains(column)) {
 +          projected.addColumn(column);
 +        }
 +      }
 +
 +    } else {
 +      // no any projected columns, meaning that all columns should be projected.
 +      // TODO - this implicit rule makes code readability bad. So, we should remove it later
 +      projected = outSchema;
 +    }
 +
 +    initScanner(projected);
      super.init();
      progress = 0.0f;
 -    if (qual != null) {
 -      qual.bind(context.getEvalContext(), inSchema);
 +
 +    if (plan.hasQual()) {
 +      if (fileScanner.isProjectable()) {
 +        qual.bind(context.getEvalContext(), projected);
 +      } else {
 +        qual.bind(context.getEvalContext(), inSchema);
 +      }
 +    }
 +  }
 +
 +  private void initScanner(Schema projected) throws IOException {
 +
-     TableMeta meta;
-     try {
-       meta = (TableMeta) plan.getTableDesc().getMeta().clone();
-     } catch (CloneNotSupportedException e) {
-       throw new RuntimeException(e);
-     }
- 
-     // set system default properties
-     PlannerUtil.applySystemDefaultToTableProperties(context.getQueryContext(), meta);
- 
 +    // Why we should check nullity? See https://issues.apache.org/jira/browse/TAJO-1422
 +    if (fragment != null) {
 +
 +      Schema fileScanOutSchema = mergeSubSchemas(projected, keySchema, plan.getTargets(),
qual);
 +
 +      this.fileScanner = OldStorageManager.getStorageManager(context.getConf(),
 +          plan.getTableDesc().getMeta().getStoreType())
 +          .getSeekableScanner(plan.getTableDesc().getMeta(), plan.getPhysicalSchema(), fragment,
fileScanOutSchema);
 +      this.fileScanner.init();
 +
 +      // See Scanner.isProjectable() method Depending on the result of isProjectable(),
 +      // the width of retrieved tuple is changed.
 +      //
 +      // If TRUE, the retrieved tuple will contain only projected fields.
 +      // If FALSE, the retrieved tuple will contain projected fields and NullDatum for non-projected
fields.
 +      if (fileScanner.isProjectable()) {
 +        this.projector = new Projector(context, projected, outSchema, plan.getTargets());
 +      } else {
 +        this.projector = new Projector(context, inSchema, outSchema, plan.getTargets());
 +      }
      }
    }
  

http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
----------------------------------------------------------------------
diff --cc tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
index 46c879c,5c2ffe3..1f878f1
--- a/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
+++ b/tajo-core/src/test/resources/results/TestTajoCli/testHelpSessionVars.result
@@@ -35,8 -35,7 +35,9 @@@ Available Session Variables
  \set MAX_OUTPUT_FILE_SIZE [int value] - Maximum per-output file size (mb). 0 means infinite.
  \set NULL_CHAR [text value] - null char of text file output
  \set CODEGEN [true or false] - Runtime code generation enabled (experiment)
 +\set INDEX_ENABLED [true or false] - index scan enabled
 +\set INDEX_SELECTIVITY_THRESHOLD [real value] - the selectivity threshold for index scan
+ \set PARTITION_NO_RESULT_OVERWRITE_ENABLED [true or false] - If True, a partitioned table
is overwritten even if a sub query leads to no result. Otherwise, the table data will be kept
if there is no result
  \set ARITHABORT [true or false] - If true, a running query will be terminated when an overflow
or divide-by-zero occurs.
  \set FETCH_ROWNUM [int value] - Sets the number of rows at a time from Master
  \set BLOCK_ON_RESULT [true or false] - Whether to block result set on query execution

http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/tajo/blob/806469a2/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java
----------------------------------------------------------------------


Mime
View raw message