tajo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hyun...@apache.org
Subject tajo git commit: TAJO-1991: Tablespace::getVolume should take filter predication.
Date Thu, 28 Jan 2016 18:14:13 GMT
Repository: tajo
Updated Branches:
  refs/heads/branch-0.11.1 0c9b6a6e9 -> 39f7b4747


TAJO-1991: Tablespace::getVolume should take filter predication.

Closes #901


Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/39f7b474
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/39f7b474
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/39f7b474

Branch: refs/heads/branch-0.11.1
Commit: 39f7b4747af960ce5c0946b26d38a64bc22a7500
Parents: 0c9b6a6
Author: Hyunsik Choi <hyunsik@apache.org>
Authored: Wed Jan 6 21:59:31 2016 -0800
Committer: Hyunsik Choi <hyunsik@apache.org>
Committed: Thu Jan 28 10:12:28 2016 -0800

----------------------------------------------------------------------
 CHANGES                                         |  3 ++
 .../org/apache/tajo/storage/StorageService.java | 42 -------------------
 .../org/apache/tajo/plan/LogicalOptimizer.java  |  6 +--
 .../org/apache/tajo/plan/LogicalPlanner.java    |  1 -
 .../org/apache/tajo/plan/StorageService.java    | 44 ++++++++++++++++++++
 .../rewrite/LogicalPlanRewriteRuleContext.java  |  2 +-
 .../plan/rewrite/TableStatUpdateRewriter.java   | 12 +++---
 .../org/apache/tajo/storage/Tablespace.java     |  3 +-
 .../apache/tajo/storage/TablespaceManager.java  |  7 +++-
 .../tajo/storage/hbase/HBaseTablespace.java     |  7 ++--
 .../org/apache/tajo/storage/FileTablespace.java |  2 +-
 tajo-storage/tajo-storage-jdbc/pom.xml          |  1 -
 .../tajo/storage/jdbc/JdbcTablespace.java       |  3 +-
 tajo-storage/tajo-storage-pgsql/pom.xml         |  2 +-
 14 files changed, 70 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index ec607d4..008b987 100644
--- a/CHANGES
+++ b/CHANGES
@@ -7,6 +7,9 @@ Release 0.11.1 - unreleased
 
   IMPROVEMENT
 
+    TAJO-1991: Tablespace::getVolume should take filter predication.
+    (hyunsik)
+
     TAJO-2007: By default, Optimizer should use the table volume in TableStat.
     (hyunsik)
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
deleted file mode 100644
index 0c3c031..0000000
--- a/tajo-common/src/main/java/org/apache/tajo/storage/StorageService.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.tajo.storage;
-
-import org.apache.tajo.exception.UnsupportedException;
-
-import javax.annotation.Nullable;
-import java.net.URI;
-
-/**
- * TablespaceManager interface for loosely coupled usages
- */
-public interface StorageService {
-
-  /**
-   * Get Table URI
-   *
-   * @param spaceName Tablespace name. If it is null, the default space will be used
-   * @param databaseName Database name
-   * @param tableName Table name
-   * @return Table URI
-   */
-  URI getTableURI(@Nullable String spaceName, String databaseName, String tableName);
-
-  long getTableVolumn(URI uri) throws UnsupportedException;
-}

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
index bb8494c..c179ed9 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java
@@ -30,11 +30,8 @@ import org.apache.tajo.algebra.JoinType;
 import org.apache.tajo.catalog.CatalogService;
 import org.apache.tajo.conf.TajoConf;
 import org.apache.tajo.conf.TajoConf.ConfVars;
-import org.apache.tajo.plan.expr.*;
 import org.apache.tajo.exception.TajoException;
-import org.apache.tajo.plan.expr.AlgebraicUtil;
-import org.apache.tajo.plan.expr.EvalNode;
-import org.apache.tajo.plan.expr.EvalTreeUtil;
+import org.apache.tajo.plan.expr.*;
 import org.apache.tajo.plan.joinorder.*;
 import org.apache.tajo.plan.logical.*;
 import org.apache.tajo.plan.rewrite.BaseLogicalPlanRewriteEngine;
@@ -42,7 +39,6 @@ import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
 import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleProvider;
 import org.apache.tajo.plan.util.PlannerUtil;
 import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
-import org.apache.tajo.storage.StorageService;
 import org.apache.tajo.util.ReflectionUtil;
 import org.apache.tajo.util.TUtil;
 import org.apache.tajo.util.graph.DirectedGraphCursor;

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
index 2bdec36..4b0caa0 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java
@@ -48,7 +48,6 @@ import org.apache.tajo.plan.nameresolver.NameResolvingMode;
 import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule;
 import org.apache.tajo.plan.util.ExprFinder;
 import org.apache.tajo.plan.util.PlannerUtil;
-import org.apache.tajo.storage.StorageService;
 import org.apache.tajo.util.KeyValueSet;
 import org.apache.tajo.util.Pair;
 import org.apache.tajo.util.StringUtils;

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java
new file mode 100644
index 0000000..f27aeff
--- /dev/null
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/StorageService.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tajo.plan;
+
+import com.google.common.base.Optional;
+import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.expr.EvalNode;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+
+/**
+ * TablespaceManager interface for loosely coupled usages
+ */
+public interface StorageService {
+
+  /**
+   * Get Table URI
+   *
+   * @param spaceName Tablespace name. If it is null, the default space will be used
+   * @param databaseName Database name
+   * @param tableName Table name
+   * @return Table URI
+   */
+  URI getTableURI(@Nullable String spaceName, String databaseName, String tableName);
+
+  long getTableVolumn(URI uri, Optional<EvalNode> filter) throws UnsupportedException;
+}

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
index 565b14b..0362001 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/LogicalPlanRewriteRuleContext.java
@@ -21,7 +21,7 @@ package org.apache.tajo.plan.rewrite;
 import org.apache.tajo.OverridableConf;
 import org.apache.tajo.catalog.CatalogService;
 import org.apache.tajo.plan.LogicalPlan;
-import org.apache.tajo.storage.StorageService;
+import org.apache.tajo.plan.StorageService;
 
 public class LogicalPlanRewriteRuleContext {
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
----------------------------------------------------------------------
diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
index 9c6f285..a1e9a6d 100644
--- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
+++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/TableStatUpdateRewriter.java
@@ -18,6 +18,7 @@
 
 package org.apache.tajo.plan.rewrite;
 
+import com.google.common.base.Optional;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.tajo.OverridableConf;
@@ -27,10 +28,11 @@ import org.apache.tajo.catalog.statistics.TableStats;
 import org.apache.tajo.exception.TajoException;
 import org.apache.tajo.exception.UnsupportedException;
 import org.apache.tajo.plan.LogicalPlan;
+import org.apache.tajo.plan.StorageService;
+import org.apache.tajo.plan.expr.EvalNode;
 import org.apache.tajo.plan.logical.LogicalNode;
 import org.apache.tajo.plan.logical.ScanNode;
 import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor;
-import org.apache.tajo.storage.StorageService;
 
 import java.util.Stack;
 
@@ -75,15 +77,15 @@ public class TableStatUpdateRewriter implements LogicalPlanRewriteRule
{
       final TableDesc table = scanNode.getTableDesc();
 
       if (!isVirtual(table)) {
-
         final TableStats stats = getTableStat(table);
         final long tableSize = stats.getNumBytes();
+        final Optional<EvalNode> filter = scanNode.hasQual() ? Optional.of(scanNode.getQual())
: Optional.<EvalNode>absent();
 
         // If USE_TABLE_VOLUME is set, we will update the table volume through a storage
handler.
         // In addition, if the table size is zero, we will update too.
         // It is a good workaround to avoid suboptimal join orders without cheap cost.
         if (conf.getBool(SessionVars.USE_TABLE_VOLUME) || tableSize == 0) {
-          table.getStats().setNumBytes(getTableVolume(table));
+          table.getStats().setNumBytes(getTableVolume(table, filter));
         }
       }
 
@@ -105,10 +107,10 @@ public class TableStatUpdateRewriter implements LogicalPlanRewriteRule
{
       return table.getMeta().getDataFormat().equals("SYSTEM");
     }
 
-    private long getTableVolume(TableDesc table) {
+    private long getTableVolume(TableDesc table, Optional<EvalNode> filter) {
       try {
         if (table.getStats() != null) {
-          return storage.getTableVolumn(table.getUri());
+          return storage.getTableVolumn(table.getUri(), filter);
         }
       } catch (UnsupportedException t) {
         LOG.warn(table.getName() + " does not support Tablespace::getTableVolume()");

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
index 81fbbc0..01ef9de 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/Tablespace.java
@@ -18,6 +18,7 @@
 
 package org.apache.tajo.storage;
 
+import com.google.common.base.Optional;
 import net.minidev.json.JSONObject;
 import org.apache.hadoop.fs.Path;
 import org.apache.tajo.ExecutionBlockId;
@@ -98,7 +99,7 @@ public abstract class Tablespace {
     return name + "=" + uri.toString();
   }
 
-  public abstract long getTableVolume(URI uri) throws UnsupportedException;
+  public abstract long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException;
 
   /**
    * if {@link StorageProperty#isArbitraryPathAllowed} is true,

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
index 668b6c6..33cd7a3 100644
--- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
+++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/TablespaceManager.java
@@ -37,6 +37,8 @@ import org.apache.tajo.exception.TajoRuntimeException;
 import org.apache.tajo.exception.UndefinedTablespaceException;
 import org.apache.tajo.exception.UndefinedTablespaceHandlerException;
 import org.apache.tajo.exception.UnsupportedException;
+import org.apache.tajo.plan.StorageService;
+import org.apache.tajo.plan.expr.EvalNode;
 import org.apache.tajo.storage.fragment.Fragment;
 import org.apache.tajo.util.JavaResourceUtil;
 import org.apache.tajo.util.Pair;
@@ -437,8 +439,9 @@ public class TablespaceManager implements StorageService {
   }
 
   @Override
-  public long getTableVolumn(URI tableUri) throws UnsupportedException {
-    return get(tableUri).getTableVolume(tableUri);
+  public long getTableVolumn(URI tableUri, Optional<EvalNode> filter)
+      throws UnsupportedException {
+    return get(tableUri).getTableVolume(tableUri, filter);
   }
 
   public static Iterable<Tablespace> getAllTablespaces() {

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
index acdb6ec..8aa51e6 100644
--- a/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
+++ b/tajo-storage/tajo-storage-hbase/src/main/java/org/apache/tajo/storage/hbase/HBaseTablespace.java
@@ -18,7 +18,7 @@
 
 package org.apache.tajo.storage.hbase;
 
-import com.google.common.base.Preconditions;
+import com.google.common.base.Optional;
 import com.google.common.collect.Sets;
 import net.minidev.json.JSONObject;
 import org.apache.commons.logging.Log;
@@ -51,7 +51,6 @@ import org.apache.tajo.plan.expr.*;
 import org.apache.tajo.plan.logical.CreateTableNode;
 import org.apache.tajo.plan.logical.LogicalNode;
 import org.apache.tajo.plan.logical.NodeType;
-import org.apache.tajo.plan.logical.ScanNode;
 import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext;
 import org.apache.tajo.plan.verifier.SyntaxErrorUtil;
 import org.apache.tajo.storage.*;
@@ -100,7 +99,7 @@ public class HBaseTablespace extends Tablespace {
   }
 
   @Override
-  public long getTableVolume(URI uri) throws UnsupportedException {
+  public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException
{
     throw new UnsupportedException();
   }
 
@@ -176,7 +175,7 @@ public class HBaseTablespace extends Tablespace {
           throw new UnavailableTableLocationException(hbaseTableName, "the table does not
exist");
         }
         HTableDescriptor hTableDescriptor = hAdmin.getTableDescriptor(hTableName);
-        Set<String> tableColumnFamilies = new HashSet<String>();
+        Set<String> tableColumnFamilies = new HashSet<>();
         for (HColumnDescriptor eachColumn : hTableDescriptor.getColumnFamilies()) {
           tableColumnFamilies.add(eachColumn.getNameAsString());
         }

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
index 9b949d6..dc4502c 100644
--- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
+++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java
@@ -127,7 +127,7 @@ public class FileTablespace extends Tablespace {
   }
 
   @Override
-  public long getTableVolume(URI uri) throws UnsupportedException {
+  public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException
{
     Path path = new Path(uri);
     ContentSummary summary;
     try {

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-storage/tajo-storage-jdbc/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/pom.xml b/tajo-storage/tajo-storage-jdbc/pom.xml
index 0b76f4d..ead7f36 100644
--- a/tajo-storage/tajo-storage-jdbc/pom.xml
+++ b/tajo-storage/tajo-storage-jdbc/pom.xml
@@ -196,7 +196,6 @@
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>
     </dependency>
-
   </dependencies>
 
   <profiles>

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
index 7489307..da0b5a7 100644
--- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
+++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcTablespace.java
@@ -18,6 +18,7 @@
 
 package org.apache.tajo.storage.jdbc;
 
+import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import net.minidev.json.JSONObject;
@@ -109,7 +110,7 @@ public abstract class JdbcTablespace extends Tablespace {
   }
 
   @Override
-  public long getTableVolume(URI uri) throws UnsupportedException {
+  public long getTableVolume(URI uri, Optional<EvalNode> filter) throws UnsupportedException
{
     throw new UnsupportedException();
   }
 

http://git-wip-us.apache.org/repos/asf/tajo/blob/39f7b474/tajo-storage/tajo-storage-pgsql/pom.xml
----------------------------------------------------------------------
diff --git a/tajo-storage/tajo-storage-pgsql/pom.xml b/tajo-storage/tajo-storage-pgsql/pom.xml
index d67abce..012d5cb 100644
--- a/tajo-storage/tajo-storage-pgsql/pom.xml
+++ b/tajo-storage/tajo-storage-pgsql/pom.xml
@@ -215,7 +215,7 @@
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
       <version>15.0</version>
-      <scope>test</scope>
+      <!--<scope>test</scope>-->
     </dependency>
   </dependencies>
 


Mime
View raw message