phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tdsi...@apache.org
Subject phoenix git commit: PHOENIX-4592 BaseResultIterators.getStatsForParallelizationProp() should use retry looking up the table without tenantId if cannot find the table using the tenantId
Date Fri, 16 Feb 2018 18:25:34 GMT
Repository: phoenix
Updated Branches:
  refs/heads/4.x-cdh5.11.2 1b14b6215 -> 65f82a9d3


PHOENIX-4592 BaseResultIterators.getStatsForParallelizationProp() should use retry looking
up the table without tenantId if cannot find the table using the tenantId


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/65f82a9d
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/65f82a9d
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/65f82a9d

Branch: refs/heads/4.x-cdh5.11.2
Commit: 65f82a9d3c9472104eacf874f0da9efedefc3444
Parents: 1b14b62
Author: Thomas D'Silva <tdsilva@apache.org>
Authored: Tue Feb 13 14:31:42 2018 -0800
Committer: Thomas D'Silva <tdsilva@apache.org>
Committed: Fri Feb 16 10:24:20 2018 -0800

----------------------------------------------------------------------
 .../end2end/ExplainPlanWithStatsEnabledIT.java  | 32 +++++++++++++
 .../coprocessor/MetaDataEndpointImpl.java       |  9 ++--
 .../phoenix/iterate/BaseResultIterators.java    | 41 ++---------------
 .../util/PhoenixConfigurationUtil.java          | 48 ++++++++++++++++++++
 4 files changed, 89 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/65f82a9d/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
index f13510b..f369be9 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ExplainPlanWithStatsEnabledIT.java
@@ -17,6 +17,7 @@
  */
 package org.apache.phoenix.end2end;
 
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FOR_PARALLELIZATION;
 import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -34,7 +35,9 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixResultSet;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
 import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.util.EnvironmentEdge;
@@ -1104,4 +1107,33 @@ public class ExplainPlanWithStatsEnabledIT extends ParallelStatsEnabledIT
{
         assertEquals("B", rs.getString(1));
     }
 
+	@Test
+	public void testUseStatsForParallelizationProperyOnViewIndex() throws SQLException {
+		String tableName = generateUniqueName();
+		String viewName = generateUniqueName();
+		String tenantViewName = generateUniqueName();
+		String viewIndexName = generateUniqueName();
+		boolean useStats = !DEFAULT_USE_STATS_FOR_PARALLELIZATION;
+		try (Connection conn = DriverManager.getConnection(getUrl())) {
+			conn.createStatement()
+					.execute("create table " + tableName
+							+ "(tenantId CHAR(15) NOT NULL, pk1 integer NOT NULL, v varchar CONSTRAINT PK PRIMARY
KEY "
+							+ "(tenantId, pk1)) MULTI_TENANT=true");
+			try (Connection tenantConn = getTenantConnection("tenant1")) {
+				conn.createStatement().execute("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableName);
+				conn.createStatement().execute("CREATE INDEX " + viewIndexName + " on " + viewName +
" (v) ");
+				tenantConn.createStatement().execute("CREATE VIEW " + tenantViewName + " AS SELECT *
FROM " + viewName);
+				conn.createStatement()
+						.execute("ALTER TABLE " + tableName + " set USE_STATS_FOR_PARALLELIZATION=" + useStats);
+				// fetch the latest view ptable 
+				PhoenixRuntime.getTableNoCache(tenantConn, viewName);
+				PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class);
+				PTable viewIndex = phxConn.getTable(new PTableKey(phxConn.getTenantId(), viewIndexName));
+				assertEquals("USE_STATS_FOR_PARALLELIZATION property set incorrectly", useStats,
+						PhoenixConfigurationUtil
+								.getStatsForParallelizationProp(tenantConn.unwrap(PhoenixConnection.class), viewIndex));
+			}
+		}
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/65f82a9d/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index d08db27..11e5e45 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -2672,9 +2672,12 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements
Coprocesso
                 for (TableProperty tableProp : TableProperty.values()) {
                     Cell tablePropertyCell = tablePropertyCellMap.get(tableProp);
                     if ( tablePropertyCell != null) {
-                        // set this table property on the view if it is not mutable on a
view (which means the property is always the same as the base table)
-                        // or if it is mutable on a view and the property value is the same
as the base table property (which means it wasn't changed on the view)
-                        if (!tableProp.isMutableOnView() || tableProp.getPTableValue(view).equals(tableProp.getPTableValue(basePhysicalTable)))
{
+						// set this table property on the view :
+						// 1. if it is not mutable on a view (which means the property is always the same as
the base table)
+						// 2. or if it is mutable on a view and if it doesn't exist on the view
+						// 3. or if it is mutable on a view and the property value is the same as the base
table property (which means it wasn't changed on the view)
+                        Object viewProp = tableProp.getPTableValue(view);
+						if (!tableProp.isMutableOnView() || viewProp==null || viewProp.equals(tableProp.getPTableValue(basePhysicalTable)))
{
                             viewHeaderRowPut.add(CellUtil.createCell(viewKey, CellUtil.cloneFamily(tablePropertyCell),
                                 CellUtil.cloneQualifier(tablePropertyCell), clientTimeStamp,
tablePropertyCell.getTypeByte(),
                                 CellUtil.cloneValue(tablePropertyCell)));

http://git-wip-us.apache.org/repos/asf/phoenix/blob/65f82a9d/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
index 25722a9..0484748 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -23,8 +23,6 @@ import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_STAR
 import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.SCAN_STOP_ROW_SUFFIX;
 import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_FAILED_QUERY_COUNTER;
 import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_QUERY_TIMEOUT_COUNTER;
-import static org.apache.phoenix.query.QueryServices.USE_STATS_FOR_PARALLELIZATION;
-import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FOR_PARALLELIZATION;
 import static org.apache.phoenix.schema.PTable.IndexType.LOCAL;
 import static org.apache.phoenix.schema.PTableType.INDEX;
 import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY;
@@ -83,8 +81,8 @@ import org.apache.phoenix.filter.DistinctPrefixFilter;
 import org.apache.phoenix.filter.EncodedQualifiersColumnProjectionFilter;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.hbase.index.util.VersionUtil;
-import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.join.HashCacheClient;
+import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil;
 import org.apache.phoenix.parse.FilterableStatement;
 import org.apache.phoenix.parse.HintNode;
 import org.apache.phoenix.parse.HintNode.Hint;
@@ -101,11 +99,9 @@ import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
 import org.apache.phoenix.schema.PTable.IndexType;
 import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.PTable.ViewType;
-import org.apache.phoenix.schema.PTableKey;
 import org.apache.phoenix.schema.PTableType;
 import org.apache.phoenix.schema.RowKeySchema;
 import org.apache.phoenix.schema.StaleRegionBoundaryCacheException;
-import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.schema.TableRef;
 import org.apache.phoenix.schema.ValueSchema.Field;
 import org.apache.phoenix.schema.stats.GuidePostsInfo;
@@ -142,7 +138,7 @@ import com.google.common.collect.Lists;
  * @since 0.1
  */
 public abstract class BaseResultIterators extends ExplainTable implements ResultIterators
{
-	private static final Logger logger = LoggerFactory.getLogger(BaseResultIterators.class);
+	public static final Logger logger = LoggerFactory.getLogger(BaseResultIterators.class);
     private static final int ESTIMATED_GUIDEPOSTS_PER_REGION = 20;
     private static final int MIN_SEEK_TO_COLUMN_VERSION = VersionUtil.encodeVersion("0",
"98", "12");
     private final List<List<Scan>> scans;
@@ -501,7 +497,7 @@ public abstract class BaseResultIterators extends ExplainTable implements
Result
         scanId = new UUID(ThreadLocalRandom.current().nextLong(), ThreadLocalRandom.current().nextLong()).toString();
         
         initializeScan(plan, perScanLimit, offset, scan);
-        this.useStatsForParallelization = getStatsForParallelizationProp(context, table);
+        this.useStatsForParallelization = PhoenixConfigurationUtil.getStatsForParallelizationProp(context.getConnection(),
table);
         this.scans = getParallelScans();
         List<KeyRange> splitRanges = Lists.newArrayListWithExpectedSize(scans.size()
* ESTIMATED_GUIDEPOSTS_PER_REGION);
         for (List<Scan> scanList : scans) {
@@ -1439,35 +1435,4 @@ public abstract class BaseResultIterators extends ExplainTable implements
Result
         return this.estimateInfoTimestamp;
     }
 
-    private boolean getStatsForParallelizationProp(StatementContext context, PTable table)
{
-        Boolean useStats = table.useStatsForParallelization();
-        if (useStats != null) {
-            return useStats;
-        }
-        /*
-         * For a view index, we use the property set on view. For indexes on base table,
whether
-         * global or local, we use the property set on the base table. Null check needed
when
-         * dropping local indexes.
-         */
-        if (table.getType() == PTableType.INDEX && table.getParentName() != null)
{
-            PhoenixConnection conn = context.getConnection();
-            String parentTableName = table.getParentName().getString();
-            try {
-                PTable parentTable =
-                        conn.getTable(new PTableKey(conn.getTenantId(), parentTableName));
-                useStats = parentTable.useStatsForParallelization();
-                if (useStats != null) {
-                    return useStats;
-                }
-            } catch (TableNotFoundException e) {
-                logger.warn("Unable to find parent table \"" + parentTableName + "\" of table
\""
-                        + table.getName().getString()
-                        + "\" to determine USE_STATS_FOR_PARALLELIZATION",
-                    e);
-            }
-        }
-        return context.getConnection().getQueryServices().getConfiguration()
-                .getBoolean(USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_PARALLELIZATION);
-    }
-
 }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/65f82a9d/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
index db11f7d..3c27f65 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtil.java
@@ -18,6 +18,8 @@
 package org.apache.phoenix.mapreduce.util;
 
 import static org.apache.commons.lang.StringUtils.isNotEmpty;
+import static org.apache.phoenix.query.QueryServices.USE_STATS_FOR_PARALLELIZATION;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FOR_PARALLELIZATION;
 
 import java.io.IOException;
 import java.sql.Connection;
@@ -37,12 +39,18 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.lib.db.DBInputFormat.NullDBWritable;
 import org.apache.hadoop.mapreduce.lib.db.DBWritable;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.phoenix.iterate.BaseResultIterators;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.mapreduce.FormatToBytesWritableMapper;
 import org.apache.phoenix.mapreduce.ImportPreUpsertKeyValueProcessor;
 import org.apache.phoenix.mapreduce.PhoenixInputFormat;
 import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.OutputFormat;
 import org.apache.phoenix.mapreduce.index.IndexScrutinyTool.SourceTable;
+import org.apache.phoenix.schema.PName;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.util.ColumnInfo;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.QueryUtil;
@@ -610,4 +618,44 @@ public final class PhoenixConfigurationUtil {
         boolean split = configuration.getBoolean(MAPREDUCE_SPLIT_BY_STATS, DEFAULT_SPLIT_BY_STATS);
         return split;
     }
+
+	public static boolean getStatsForParallelizationProp(PhoenixConnection conn, PTable table)
{
+	    Boolean useStats = table.useStatsForParallelization();
+	    if (useStats != null) {
+	        return useStats;
+	    }
+	    /*
+	     * For a view index, we use the property set on view. For indexes on base table, whether
+	     * global or local, we use the property set on the base table. Null check needed when
+	     * dropping local indexes.
+	     */
+	    PName tenantId = conn.getTenantId();
+	    int retryCount = 0;
+	    while (retryCount++<2) {
+		    if (table.getType() == PTableType.INDEX && table.getParentName() != null) {
+		        String parentTableName = table.getParentName().getString();
+				try {
+		            PTable parentTable =
+		                    conn.getTable(new PTableKey(tenantId, parentTableName));
+		            useStats = parentTable.useStatsForParallelization();
+		            if (useStats != null) {
+		                return useStats;
+		            }
+				} catch (TableNotFoundException e) {
+					// try looking up the table without the tenant id (for
+					// global tables)
+					if (tenantId != null) {
+						tenantId = null;
+					} else {
+						BaseResultIterators.logger.warn(
+								"Unable to find parent table \"" + parentTableName + "\" of table \""
+										+ table.getName().getString() + "\" to determine USE_STATS_FOR_PARALLELIZATION",
+								e);
+					}
+				}
+		    }
+	    }
+	    return conn.getQueryServices().getConfiguration()
+	            .getBoolean(USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_PARALLELIZATION);
+	}
 }


Mime
View raw message