phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maryann...@apache.org
Subject phoenix git commit: 1. Retrieve PhoenixTable stats the same way as PhoenixTableScan stats; 2. Handle no stats cases; 3. Revert temp changes for PHOENIX-2712
Date Fri, 26 Feb 2016 14:14:34 GMT
Repository: phoenix
Updated Branches:
  refs/heads/calcite 36ee23ab7 -> a78fdf5b6


1. Retrieve PhoenixTable stats the same way as PhoenixTableScan stats; 2. Handle no stats
cases; 3. Revert temp changes for PHOENIX-2712


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a78fdf5b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a78fdf5b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a78fdf5b

Branch: refs/heads/calcite
Commit: a78fdf5b6db7b8cf1016811dc4872b5e70c4b748
Parents: 36ee23a
Author: maryannxue <maryann.xue@gmail.com>
Authored: Fri Feb 26 09:14:05 2016 -0500
Committer: maryannxue <maryann.xue@gmail.com>
Committed: Fri Feb 26 09:14:05 2016 -0500

----------------------------------------------------------------------
 .../apache/phoenix/calcite/PhoenixTable.java    | 51 ++++++++------------
 .../phoenix/calcite/rel/PhoenixTableScan.java   | 49 ++++++++++---------
 .../coprocessor/MetaDataEndpointImpl.java       | 18 ++++---
 .../apache/phoenix/schema/MetaDataClient.java   |  3 --
 4 files changed, 55 insertions(+), 66 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/a78fdf5b/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java b/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java
index 101ee5d..47ac80a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/PhoenixTable.java
@@ -20,22 +20,27 @@ import org.apache.calcite.schema.TranslatableTable;
 import org.apache.calcite.schema.impl.AbstractTable;
 import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.calcite.rel.PhoenixTableScan;
+import org.apache.phoenix.compile.ColumnResolver;
+import org.apache.phoenix.compile.FromCompiler;
+import org.apache.phoenix.compile.SequenceManager;
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.iterate.BaseResultIterators;
 import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
 import org.apache.phoenix.schema.PColumn;
-import org.apache.phoenix.schema.PColumnFamily;
 import org.apache.phoenix.schema.PTable;
-import org.apache.phoenix.schema.RowKeySchema;
 import org.apache.phoenix.schema.SortOrder;
-import org.apache.phoenix.schema.stats.GuidePostsInfo;
+import org.apache.phoenix.schema.TableRef;
 import org.apache.phoenix.schema.stats.StatisticsUtil;
 import org.apache.phoenix.schema.types.PDataType;
 import org.apache.phoenix.util.SchemaUtil;
-import org.apache.phoenix.util.SizedUtil;
-
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -89,12 +94,18 @@ public class PhoenixTable extends AbstractTable implements TranslatableTable
{
       }
       this.pkBitSet = ImmutableBitSet.of(pkPositions);
       this.collation = RelCollationTraitDef.INSTANCE.canonize(RelCollations.of(fieldCollations));
-      byte[] emptyCf = SchemaUtil.getEmptyColumnFamily(pTable);
-      GuidePostsInfo info = pTable.getTableStats().getGuidePosts().get(emptyCf);
       long rowCount = 0;
       long byteCount = 0;
       try {
-          if (info == null) {
+          PhoenixStatement stmt = new PhoenixStatement(pc);
+          TableRef tableRef = new TableRef(CalciteUtils.createTempAlias(), pTable, HConstants.LATEST_TIMESTAMP,
false);
+          ColumnResolver resolver = FromCompiler.getResolver(tableRef);
+          StatementContext context = new StatementContext(stmt, resolver, new Scan(), new
SequenceManager(stmt));
+          Pair<Long, Long> estimatedCount = BaseResultIterators.getEstimatedCount(context,
pTable);
+          if (estimatedCount != null) {
+              rowCount = estimatedCount.getFirst();
+              byteCount = estimatedCount.getSecond();
+          } else {
               // TODO The props might not be the same as server props.
               int guidepostPerRegion = pc.getQueryServices().getProps().getInt(
                       QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
@@ -109,30 +120,8 @@ public class PhoenixTable extends AbstractTable implements TranslatableTable
{
               }
               byteCount = StatisticsUtil.getGuidePostDepth(
                       guidepostPerRegion, guidepostWidth, desc) / 2;
-              long keySize = pTable.getRowKeySchema().getEstimatedByteSize();
-              long rowSize = 0;
-              for (PColumnFamily cf : pTable.getColumnFamilies()) {
-                  for (PColumn column : cf.getColumns()) {
-                      Integer maxLength = column.getMaxLength();
-                      int byteSize = column.getDataType().isFixedWidth() ?
-                              maxLength == null ?
-                                      column.getDataType().getByteSize()
-                                      : maxLength
-                                      : RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE;
-                      rowSize += SizedUtil.KEY_VALUE_SIZE + keySize + byteSize;
-                  }
-              }
-              if (rowSize == 0) {
-                  rowSize = keySize;
-              }
+              long rowSize = SchemaUtil.estimateRowSize(pTable);
               rowCount = byteCount / rowSize;
-          } else {
-              for (long b : info.getByteCounts()) {
-                  byteCount += b;
-              }
-              for (long r : info.getRowCounts()) {
-                  rowCount += r;
-              }
           }
       } catch (SQLException | IOException e) {
           throw new RuntimeException(e);

http://git-wip-us.apache.org/repos/asf/phoenix/blob/a78fdf5b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java
b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java
index dfdb507..691b5ac 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/calcite/rel/PhoenixTableScan.java
@@ -65,7 +65,7 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel {
     public final ScanOrder scanOrder;
     public final ScanRanges scanRanges;
     
-    protected final long estimatedBytes;
+    protected final Long estimatedBytes;
     protected final float rowCountFactor;
     
     public static PhoenixTableScan create(RelOptCluster cluster, final RelOptTable table)
{
@@ -96,15 +96,16 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel
{
         this.scanOrder = scanOrder;
         final PhoenixTable phoenixTable = table.unwrap(PhoenixTable.class);
         this.rowCountFactor = phoenixTable.pc.getQueryServices()
-                .getProps().getFloat(PhoenixRel.ROW_COUNT_FACTOR, 1f);
-        
-        ScanRanges scanRanges = null;
-        long estimatedSize = 0;
-        if (filter != null) {
-            try {
-                // TODO simplify this code
-                PTable pTable = phoenixTable.getTable();
-                TableRef tableRef = new TableRef(CalciteUtils.createTempAlias(), pTable,
HConstants.LATEST_TIMESTAMP, false);
+                .getProps().getFloat(PhoenixRel.ROW_COUNT_FACTOR, 1f);        
+        try {
+            // TODO simplify this code
+            PTable pTable = phoenixTable.getTable();
+            TableRef tableRef = new TableRef(CalciteUtils.createTempAlias(), pTable, HConstants.LATEST_TIMESTAMP,
false);
+            SelectStatement select = SelectStatement.SELECT_ONE;
+            PhoenixStatement stmt = new PhoenixStatement(phoenixTable.pc);
+            ColumnResolver resolver = FromCompiler.getResolver(tableRef);
+            StatementContext context = new StatementContext(stmt, resolver, new Scan(), new
SequenceManager(stmt));
+            if (filter != null) {
                 // We use a implementor with a special implementation for field access
                 // here, which translates RexFieldAccess into a LiteralExpression
                 // with a sample value. This will achieve 3 goals at a time:
@@ -124,21 +125,15 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel
{
                     }                    
                 };
                 tmpImplementor.setTableRef(tableRef);
-                SelectStatement select = SelectStatement.SELECT_ONE;
-                PhoenixStatement stmt = new PhoenixStatement(phoenixTable.pc);
-                ColumnResolver resolver = FromCompiler.getResolver(tableRef);
-                StatementContext context = new StatementContext(stmt, resolver, new Scan(),
new SequenceManager(stmt));
                 Expression filterExpr = CalciteUtils.toExpression(filter, tmpImplementor);
                 filterExpr = WhereOptimizer.pushKeyExpressionsToScan(context, select, filterExpr);
                 WhereCompiler.setScanFilter(context, select, filterExpr, true, false);
-                scanRanges = context.getScanRanges();
-                estimatedSize = BaseResultIterators.getEstimatedCount(context, tableRef.getTable()).getSecond();
-            } catch (SQLException e) {
-                throw new RuntimeException(e);
-            }
-        }        
-        this.scanRanges = scanRanges;
-        this.estimatedBytes = estimatedSize;
+            }        
+            this.scanRanges = context.getScanRanges();
+            this.estimatedBytes = BaseResultIterators.getEstimatedCount(context, pTable).getSecond();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
     }
     
     private static ScanOrder getDefaultScanOrder(PhoenixTable table) {
@@ -182,10 +177,16 @@ public class PhoenixTableScan extends TableScan implements PhoenixRel
{
     public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
         double byteCount;
         PhoenixTable phoenixTable = table.unwrap(PhoenixTable.class);
-        if (scanRanges != null) {
+        if (estimatedBytes != null) {
             byteCount = estimatedBytes;
         } else {
-            byteCount = phoenixTable.byteCount;
+            // If stats are not available, we estimate based on selectivity.
+            int pkCount = scanRanges.getBoundPkColumnCount();
+            if (pkCount > 0) {
+                byteCount = phoenixTable.byteCount * Math.pow(mq.getSelectivity(this, filter),
pkCount);
+            } else {
+                byteCount = phoenixTable.byteCount;
+            }
         }
         byteCount *= rowCountFactor;
         if (scanOrder != ScanOrder.NONE) {

http://git-wip-us.apache.org/repos/asf/phoenix/blob/a78fdf5b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 7b19515..008e1d9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -869,14 +869,16 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements
Coprocesso
                 schemaName.getString(), tableName.getString())) : physicalTables.get(0);
         PTableStats stats = PTableStats.EMPTY_STATS;
         HTableInterface statsHTable = null;
-        try {
-            statsHTable = ServerUtil.getHTableForCoprocessorScan(env, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES);
-            stats = StatisticsUtil.readStatistics(statsHTable, physicalTableName.getBytes(),
clientTimeStamp);
-            timeStamp = Math.max(timeStamp, stats.getTimestamp());
-        } catch (org.apache.hadoop.hbase.TableNotFoundException e) {
-            logger.warn(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " not online yet?");
-        } finally {
-            if (statsHTable != null) statsHTable.close();
+        if (tenantId == null) {
+            try {
+                statsHTable = ServerUtil.getHTableForCoprocessorScan(env, PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES);
+                stats = StatisticsUtil.readStatistics(statsHTable, physicalTableName.getBytes(),
clientTimeStamp);
+                timeStamp = Math.max(timeStamp, stats.getTimestamp());
+            } catch (org.apache.hadoop.hbase.TableNotFoundException e) {
+                logger.warn(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME + " not online yet?");
+            } finally {
+                if (statsHTable != null) statsHTable.close();
+            }
         }
         return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState,
timeStamp,
             tableSeqNum, pkName, saltBucketNum, columns, tableType == INDEX ? schemaName
: null,

http://git-wip-us.apache.org/repos/asf/phoenix/blob/a78fdf5b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 7e28933..7f3f850 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -963,9 +963,6 @@ public class MetaDataClient {
          *  This supports scenarios in which a major compaction was manually initiated and
the
          *  client wants the modified stats to be reflected immediately.
          */
-        connection.getQueryServices().clearTableFromCache(logicalTable.getTenantId() == null
? ByteUtil.EMPTY_BYTE_ARRAY : logicalTable.getTenantId().getBytes(),
-                Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(logicalTable.getName().toString())),
-                Bytes.toBytes(SchemaUtil.getTableNameFromFullName(logicalTable.getName().toString())),
clientTimeStamp);
         connection.getQueryServices().clearTableFromCache(tenantIdBytes,
                 Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(physicalName.getString())),
                 Bytes.toBytes(SchemaUtil.getTableNameFromFullName(physicalName.getString())),
clientTimeStamp);


Mime
View raw message