Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 547FE200BB8 for ; Fri, 28 Oct 2016 23:33:58 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 5340D160B08; Fri, 28 Oct 2016 21:33:58 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C636E160B03 for ; Fri, 28 Oct 2016 23:33:55 +0200 (CEST) Received: (qmail 71931 invoked by uid 500); 28 Oct 2016 21:33:54 -0000 Mailing-List: contact commits-help@phoenix.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@phoenix.apache.org Delivered-To: mailing list commits@phoenix.apache.org Received: (qmail 71706 invoked by uid 99); 28 Oct 2016 21:33:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 28 Oct 2016 21:33:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3A658EFE5E; Fri, 28 Oct 2016 21:33:54 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: maryannxue@apache.org To: commits@phoenix.apache.org Date: Fri, 28 Oct 2016 21:33:56 -0000 Message-Id: <6204fd06e6984bf992ab6075b919e2c6@git.apache.org> In-Reply-To: <4e178594d5ad496cacb2eac93a560830@git.apache.org> References: <4e178594d5ad496cacb2eac93a560830@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [03/35] phoenix git commit: PHOENIX-2675 Allow stats to be configured on a table-by-table basis archived-at: Fri, 28 Oct 2016 21:33:58 -0000 PHOENIX-2675 Allow stats to be configured on a table-by-table basis Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/711d7f06 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/711d7f06 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/711d7f06 Branch: refs/heads/calcite Commit: 711d7f0639d294974c2f56c7d604bc37c22e0a95 Parents: ad99952 Author: James Taylor Authored: Tue Oct 4 10:34:21 2016 -0700 Committer: James Taylor Committed: Fri Oct 7 17:19:15 2016 -0700 ---------------------------------------------------------------------- .../apache/phoenix/end2end/GroupByCaseIT.java | 33 ++++- .../phoenix/end2end/index/ViewIndexIT.java | 129 +++++++++++++++---- .../apache/phoenix/compile/PostDDLCompiler.java | 8 +- .../phoenix/coprocessor/MetaDataProtocol.java | 8 +- .../UngroupedAggregateRegionObserver.java | 13 +- .../query/ConnectionQueryServicesImpl.java | 67 +++++++++- .../apache/phoenix/query/QueryConstants.java | 1 + .../apache/phoenix/schema/MetaDataClient.java | 71 ++++++---- .../apache/phoenix/schema/TableProperty.java | 7 + .../stats/DefaultStatisticsCollector.java | 109 ++++++++++++---- .../schema/stats/NoOpStatisticsCollector.java | 6 +- .../schema/stats/StatisticsCollector.java | 7 +- .../stats/StatisticsCollectorFactory.java | 19 +-- .../phoenix/schema/stats/StatisticsScanner.java | 3 +- .../phoenix/schema/stats/StatisticsUtil.java | 10 +- .../java/org/apache/phoenix/util/IndexUtil.java | 2 +- .../org/apache/phoenix/util/MetaDataUtil.java | 40 +++++- .../org/apache/phoenix/util/UpgradeUtil.java | 2 +- .../phoenix/compile/QueryCompilerTest.java | 16 +++ 19 files changed, 429 insertions(+), 122 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java index be59fd7..56f4503 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java @@ -23,20 +23,24 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; import java.util.Properties; +import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.AmbiguousColumnException; import org.apache.phoenix.schema.types.PChar; import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.util.ByteUtil; import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; +import org.apache.phoenix.util.TestUtil; import org.junit.Test; @@ -401,12 +405,29 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT { } @Test - public void testAvgGroupByOrderPreserving() throws Exception { + public void testAvgGroupByOrderPreservingWithStats() throws Exception { Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); Connection conn = DriverManager.getConnection(getUrl(), props); String tableName = generateUniqueName(); - - PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) split on (?,?,?)"); + initAvgGroupTable(conn, tableName, " GUIDE_POST_WIDTH=20 "); + testAvgGroupByOrderPreserving(conn, tableName, 13); + conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=" + 100); + testAvgGroupByOrderPreserving(conn, tableName, 6); + conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=null"); + testAvgGroupByOrderPreserving(conn, tableName, 4); + } + + @Test + public void testAvgGroupByOrderPreservingWithNoStats() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + Connection conn = DriverManager.getConnection(getUrl(), props); + String tableName = generateUniqueName(); + initAvgGroupTable(conn, tableName, ""); + testAvgGroupByOrderPreserving(conn, tableName, 4); + } + + private void initAvgGroupTable(Connection conn, String tableName, String tableProps) throws SQLException { + PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) " + tableProps + " split on (?,?,?)"); stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3))); stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3))); stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3))); @@ -425,6 +446,9 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT { conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)"); conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)"); conn.commit(); + } + + private void testAvgGroupByOrderPreserving(Connection conn, String tableName, int nGuidePosts) throws SQLException, IOException { String query = "SELECT k1,avg(k2) FROM " + tableName + " GROUP BY k1"; ResultSet rs = conn.createStatement().executeQuery(query); assertTrue(rs.next()); @@ -445,6 +469,9 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT { "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName + "\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", QueryUtil.getExplainPlan(rs)); + TestUtil.analyzeTable(conn, tableName); + List splits = TestUtil.getAllSplits(conn, tableName); + assertEquals(nGuidePosts, splits.size()); } @Test http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java index ca50aab..b714a11 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java @@ -38,7 +38,11 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.schema.PNameFactory; +import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.util.PropertiesUtil; import org.apache.phoenix.util.QueryUtil; +import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.TestUtil; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -59,7 +63,7 @@ public class ViewIndexIT extends ParallelStatsDisabledIT { if (isNamespaceMapped) { conn.createStatement().execute("CREATE SCHEMA IF NOT EXISTS " + schemaName); } - String ddl = "CREATE TABLE " + tableName + " (t_id VARCHAR NOT NULL,\n" + + String ddl = "CREATE TABLE " + SchemaUtil.getTableName(schemaName, tableName) + " (t_id VARCHAR NOT NULL,\n" + "k1 VARCHAR NOT NULL,\n" + "k2 INTEGER NOT NULL,\n" + "v1 VARCHAR,\n" + @@ -93,30 +97,31 @@ public class ViewIndexIT extends ParallelStatsDisabledIT { @Test public void testDeleteViewIndexSequences() throws Exception { String schemaName = generateUniqueName(); - String tableName = schemaName + "." + generateUniqueName(); + String tableName = generateUniqueName(); + String fullTableName = SchemaUtil.getTableName(schemaName, tableName); String indexName = "IND_" + generateUniqueName(); - String VIEW_NAME = "VIEW_" + generateUniqueName(); - String viewName = schemaName + "." + VIEW_NAME; + String viewName = "VIEW_" + generateUniqueName(); + String fullViewName = SchemaUtil.getTableName(schemaName, viewName); createBaseTable(schemaName, tableName, false, null, null); Connection conn1 = getConnection(); Connection conn2 = getConnection(); - conn1.createStatement().execute("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableName); - conn1.createStatement().execute("CREATE INDEX " + indexName + " ON " + viewName + " (v1)"); - conn2.createStatement().executeQuery("SELECT * FROM " + tableName).next(); - String sequenceName = getViewIndexSequenceName(PNameFactory.newName(tableName), null, isNamespaceMapped); - String sequenceSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(tableName), isNamespaceMapped); - String seqName = getViewIndexSequenceName(PNameFactory.newName(tableName), null, !isNamespaceMapped); - String seqSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(tableName), !isNamespaceMapped); + conn1.createStatement().execute("CREATE VIEW " + fullViewName + " AS SELECT * FROM " + fullTableName); + conn1.createStatement().execute("CREATE INDEX " + indexName + " ON " + fullViewName + " (v1)"); + conn2.createStatement().executeQuery("SELECT * FROM " + fullTableName).next(); + String sequenceName = getViewIndexSequenceName(PNameFactory.newName(fullTableName), null, isNamespaceMapped); + String sequenceSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(fullTableName), isNamespaceMapped); + String seqName = getViewIndexSequenceName(PNameFactory.newName(fullTableName), null, !isNamespaceMapped); + String seqSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(fullTableName), !isNamespaceMapped); verifySequenceValue(null, sequenceName, sequenceSchemaName, -32767); verifySequenceValue(null, sequenceName, sequenceSchemaName, -32767); - conn1.createStatement().execute("CREATE INDEX " + indexName + "_2 ON " + viewName + " (v1)"); + conn1.createStatement().execute("CREATE INDEX " + indexName + "_2 ON " + fullViewName + " (v1)"); verifySequenceValue(null, sequenceName, sequenceSchemaName, -32766); // Check other format of sequence is not there as Sequences format is different for views/indexes created on // table which are namespace mapped and which are not. verifySequenceNotExists(null, seqName, seqSchemaName); - conn1.createStatement().execute("DROP VIEW " + viewName); - conn1.createStatement().execute("DROP TABLE "+ tableName); + conn1.createStatement().execute("DROP VIEW " + fullViewName); + conn1.createStatement().execute("DROP TABLE "+ fullTableName); verifySequenceNotExists(null, sequenceName, sequenceSchemaName); } @@ -126,11 +131,13 @@ public class ViewIndexIT extends ParallelStatsDisabledIT { String schemaName = generateUniqueName(); String tableName = generateUniqueName(); String indexName = "IND_" + generateUniqueName(); - String VIEW_NAME = "VIEW_" + generateUniqueName(); + String viewName = "VIEW_" + generateUniqueName(); + String fullTableName = SchemaUtil.getTableName(schemaName, tableName); + createBaseTable(schemaName, tableName, true, null, null); Connection conn = DriverManager.getConnection(getUrl()); PreparedStatement stmt = conn.prepareStatement( - "UPSERT INTO " + tableName + "UPSERT INTO " + fullTableName + " VALUES(?,?,?,?,?)"); stmt.setString(1, "10"); stmt.setString(2, "a"); @@ -144,30 +151,61 @@ public class ViewIndexIT extends ParallelStatsDisabledIT { stmt.setString(4, "x2"); stmt.setInt(5, 200); stmt.execute(); + stmt.setString(1, "30"); + stmt.setString(2, "c"); + stmt.setInt(3, 3); + stmt.setString(4, "x3"); + stmt.setInt(5, 300); + stmt.execute(); + stmt.setString(1, "40"); + stmt.setString(2, "d"); + stmt.setInt(3, 4); + stmt.setString(4, "x4"); + stmt.setInt(5, 400); + stmt.execute(); conn.commit(); - conn.close(); - Properties props = new Properties(); + Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES); props.setProperty("TenantId", "10"); Connection conn1 = DriverManager.getConnection(getUrl(), props); - conn1.createStatement().execute("CREATE VIEW " + VIEW_NAME - + " AS select * from " + tableName); + conn1.createStatement().execute("CREATE VIEW " + viewName + + " AS select * from " + fullTableName); conn1.createStatement().execute("CREATE LOCAL INDEX " + indexName + " ON " - + VIEW_NAME + "(v2)"); + + viewName + "(v2)"); conn1.commit(); - String sql = "SELECT * FROM " + VIEW_NAME + " WHERE v2 = 100"; + String sql = "SELECT * FROM " + viewName + " WHERE v2 = 100"; ResultSet rs = conn1.prepareStatement("EXPLAIN " + sql).executeQuery(); assertEquals( - "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " [1,'10',100]\n" + + "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + SchemaUtil.getPhysicalHBaseTableName(fullTableName, isNamespaceMapped, PTableType.TABLE) + " [1,'10',100]\n" + " SERVER FILTER BY FIRST KEY ONLY\n" + "CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs)); rs = conn1.prepareStatement(sql).executeQuery(); assertTrue(rs.next()); assertFalse(rs.next()); + +// TestUtil.analyzeTable(conn, fullTableName); +// List guideposts = TestUtil.getAllSplits(conn, fullTableName); +// assertEquals(1, guideposts.size()); +// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0)); +// +// conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET GUIDE_POST_WIDTH=20"); +// +// TestUtil.analyzeTable(conn, fullTableName); +// guideposts = TestUtil.getAllSplits(conn, fullTableName); +// assertEquals(5, guideposts.size()); +// +// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table +// // was used +// sql = "SELECT * FROM " + viewName + " WHERE v2 > 100"; +// stmt = conn1.prepareStatement(sql); +// stmt.executeQuery(); +// QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan(); +// assertEquals(5, plan.getSplits().size()); } + @Test public void testCreatingIndexOnGlobalView() throws Exception { String baseTable = generateUniqueName(); @@ -184,6 +222,30 @@ public class ViewIndexIT extends ParallelStatsDisabledIT { stmt.setString(4, "KV1"); stmt.setString(5, "KV3"); stmt.executeUpdate(); + stmt.setString(1, "tenantId"); + stmt.setDate(2, new Date(100)); + stmt.setInt(3, 2); + stmt.setString(4, "KV4"); + stmt.setString(5, "KV5"); + stmt.executeUpdate(); + stmt.setString(1, "tenantId"); + stmt.setDate(2, new Date(100)); + stmt.setInt(3, 3); + stmt.setString(4, "KV6"); + stmt.setString(5, "KV7"); + stmt.executeUpdate(); + stmt.setString(1, "tenantId"); + stmt.setDate(2, new Date(100)); + stmt.setInt(3, 4); + stmt.setString(4, "KV8"); + stmt.setString(5, "KV9"); + stmt.executeUpdate(); + stmt.setString(1, "tenantId"); + stmt.setDate(2, new Date(100)); + stmt.setInt(3, 5); + stmt.setString(4, "KV10"); + stmt.setString(5, "KV11"); + stmt.executeUpdate(); conn.commit(); // Verify that query against the global view index works @@ -196,6 +258,27 @@ public class ViewIndexIT extends ParallelStatsDisabledIT { assertTrue(rs.next()); assertEquals("KV1", rs.getString(1)); assertFalse(rs.next()); + +// TestUtil.analyzeTable(conn, baseTable); +// List guideposts = TestUtil.getAllSplits(conn, baseTable); +// assertEquals(1, guideposts.size()); +// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0)); +// +// conn.createStatement().execute("ALTER TABLE " + baseTable + " SET GUIDE_POST_WIDTH=20"); +// +// TestUtil.analyzeTable(conn, baseTable); +// guideposts = TestUtil.getAllSplits(conn, baseTable); +// assertEquals(6, guideposts.size()); +// +// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table +// // was used +// stmt = conn.prepareStatement("SELECT KV1 FROM " + globalView + " WHERE PK3 = ? AND KV3 >= ?"); +// stmt.setInt(1, 1); +// stmt.setString(2, "KV3"); +// rs = stmt.executeQuery(); +// plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan(); +// assertTrue(plan.getTableRef().getTable().getName().getString().equals(globalViewIdx)); +// assertEquals(6, plan.getSplits().size()); } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java index e43b596..004e254 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java @@ -82,7 +82,7 @@ public class PostDDLCompiler { scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE); } - public MutationPlan compile(final List tableRefs, final byte[] emptyCF, final byte[] projectCF, final List deleteList, + public MutationPlan compile(final List tableRefs, final byte[] emptyCF, final List projectCFs, final List deleteList, final long timestamp) throws SQLException { PhoenixStatement statement = new PhoenixStatement(connection); final StatementContext context = new StatementContext( @@ -244,12 +244,14 @@ public class PostDDLCompiler { } } List columnFamilies = Lists.newArrayListWithExpectedSize(tableRef.getTable().getColumnFamilies().size()); - if (projectCF == null) { + if (projectCFs == null) { for (PColumnFamily family : tableRef.getTable().getColumnFamilies()) { columnFamilies.add(family.getName().getBytes()); } } else { - columnFamilies.add(projectCF); + for (byte[] projectCF : projectCFs) { + columnFamilies.add(projectCF); + } } // Need to project all column families into the scan, since we haven't yet created our empty key value RowProjector projector = ProjectionCompiler.compile(context, SelectStatement.COUNT_ONE, GroupBy.EMPTY_GROUP_BY); http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java index bb720ee..4f0a34c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java @@ -62,8 +62,8 @@ import com.google.protobuf.ByteString; */ public abstract class MetaDataProtocol extends MetaDataService { public static final int PHOENIX_MAJOR_VERSION = 4; - public static final int PHOENIX_MINOR_VERSION = 8; - public static final int PHOENIX_PATCH_NUMBER = 1; + public static final int PHOENIX_MINOR_VERSION = 9; + public static final int PHOENIX_PATCH_NUMBER = 0; public static final int PHOENIX_VERSION = VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER); @@ -84,8 +84,9 @@ public abstract class MetaDataProtocol extends MetaDataService { public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 15; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 = MIN_TABLE_TIMESTAMP + 18; public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_1 = MIN_TABLE_TIMESTAMP + 18; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0 = MIN_TABLE_TIMESTAMP + 19; // MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants - public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_8_1; + public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0; // ALWAYS update this map whenever rolling out a new release (major, minor or patch release). // Key is the SYSTEM.CATALOG timestamp for the version and value is the version string. @@ -99,6 +100,7 @@ public abstract class MetaDataProtocol extends MetaDataService { TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, "4.6.x"); TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0, "4.7.x"); TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0, "4.8.x"); + TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0, "4.9.x"); } public static final String CURRENT_CLIENT_VERSION = PHOENIX_MAJOR_VERSION + "." + PHOENIX_MINOR_VERSION + "." + PHOENIX_PATCH_NUMBER; http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java index a7c6fde..2a4bfca 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java @@ -49,10 +49,8 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.RegionTooBusyException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Scan; @@ -113,6 +111,7 @@ import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.ServerUtil; import org.apache.phoenix.util.StringUtil; import org.apache.phoenix.util.TimeKeeper; +import org.apache.tephra.TxConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -120,8 +119,6 @@ import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.tephra.TxConstants; - /** * Region observer that aggregates ungrouped rows(i.e. SQL query with aggregation function and no GROUP BY). @@ -229,9 +226,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver throws IOException { s = super.preScannerOpen(e, scan, s); if (ScanUtil.isAnalyzeTable(scan)) { - if (!ScanUtil.isLocalIndex(scan)) { - scan.getFamilyMap().clear(); - } +// if (!ScanUtil.isLocalIndex(scan)) { +// scan.getFamilyMap().clear(); +// } +// scan.getFamilyMap().clear(); // We are setting the start row and stop row such that it covers the entire region. As part // of Phonenix-1263 we are storing the guideposts against the physical table rather than // individual tenant specific tables. @@ -815,6 +813,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver long rowCount = 0; try { if (!compactionRunning) { + stats.init(); synchronized (innerScanner) { do { List results = new ArrayList(); http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 354a932..4982578 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -36,6 +36,7 @@ import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0; import java.io.IOException; import java.lang.ref.WeakReference; +import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; @@ -2257,6 +2258,41 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement } + private PhoenixConnection removeNotNullConstraint(PhoenixConnection oldMetaConnection, String schemaName, String tableName, long timestamp, String columnName) throws SQLException { + Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo()); + props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp)); + // Cannot go through DriverManager or you end up in an infinite loop because it'll call init again + PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props); + SQLException sqlE = null; + try { + metaConnection.createStatement().executeUpdate("UPSERT INTO " + SYSTEM_STATS_NAME + " (" + + PhoenixDatabaseMetaData.TENANT_ID + "," + PhoenixDatabaseMetaData.TABLE_SCHEM + "," + + PhoenixDatabaseMetaData.TABLE_NAME + "," + PhoenixDatabaseMetaData.COLUMN_NAME + "," + + PhoenixDatabaseMetaData.COLUMN_FAMILY + "," + PhoenixDatabaseMetaData.NULLABLE + ") VALUES (" + + "null," + schemaName + "," + tableName + "," + columnName + "," + QueryConstants.DEFAULT_COLUMN_FAMILY + "," + + ResultSetMetaData.columnNullable + ")"); + metaConnection.commit(); + } catch (NewerTableAlreadyExistsException e) { + logger.warn("Table already modified at this timestamp, so assuming column already nullable: " + columnName); + } catch (SQLException e) { + logger.warn("Add column failed due to:" + e); + sqlE = e; + } finally { + try { + oldMetaConnection.close(); + } catch (SQLException e) { + if (sqlE != null) { + sqlE.setNextException(e); + } else { + sqlE = e; + } + } + if (sqlE != null) { + throw sqlE; + } + } + return metaConnection; + } /** * This closes the passed connection. */ @@ -2474,7 +2510,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null && table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) { table.setValue(MetaDataUtil.PARENT_TABLE_KEY, - MetaDataUtil.getUserTableName(table.getNameAsString())); + MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString())); // Explicitly disable, modify and enable the table to ensure // co-location of data and index regions. If we just modify the // table descriptor when online schema change enabled may reopen @@ -2620,6 +2656,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0); clearCache(); } + if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) { + metaConnection = addColumnsIfNotExists( + metaConnection, + PhoenixDatabaseMetaData.SYSTEM_CATALOG, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0, + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " " + + PLong.INSTANCE.getSqlTypeName()); + ConnectionQueryServicesImpl.this.removeTable(null, + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0); + clearCache(); + } } @@ -2676,7 +2724,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement try { metaConnection.createStatement().executeUpdate( QueryConstants.CREATE_STATS_TABLE_METADATA); - } catch (NewerTableAlreadyExistsException ignore) {} catch (TableAlreadyExistsException e) { + } catch (NewerTableAlreadyExistsException ignore) { + + } catch (TableAlreadyExistsException e) { long currentServerSideTableTimeStamp = e.getTable().getTimeStamp(); if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) { metaConnection = addColumnsIfNotExists( @@ -2686,6 +2736,19 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT + " " + PLong.INSTANCE.getSqlTypeName()); } + if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) { + // The COLUMN_FAMILY column should be nullable as we create a row in it without + // any column family to mark when guideposts were last collected. + metaConnection = removeNotNullConstraint(metaConnection, + PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME, + PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0, + PhoenixDatabaseMetaData.COLUMN_FAMILY); + ConnectionQueryServicesImpl.this.removeTable(null, + PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, null, + MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0); + clearCache(); + } } try { metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA); http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java index ace228b..fd37328 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java @@ -288,6 +288,7 @@ public interface QueryConstants { IS_NAMESPACE_MAPPED + " BOOLEAN," + AUTO_PARTITION_SEQ + " VARCHAR," + APPEND_ONLY_SCHEMA + " BOOLEAN," + + GUIDE_POSTS_WIDTH + " BIGINT," + "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" + HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" + http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 1e635d8..0c53d3e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -42,6 +42,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAM import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE; @@ -256,8 +257,9 @@ public class MetaDataClient { UPDATE_CACHE_FREQUENCY + "," + IS_NAMESPACE_MAPPED + "," + AUTO_PARTITION_SEQ + "," + - APPEND_ONLY_SCHEMA + - ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + APPEND_ONLY_SCHEMA + "," + + GUIDE_POSTS_WIDTH + + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; private static final String CREATE_SCHEMA = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TABLE_SCHEM + "," + TABLE_NAME + ") VALUES (?,?)"; @@ -993,30 +995,20 @@ public class MetaDataClient { // If analyzing the indexes of a multi-tenant table or a table with view indexes // then analyze all of those indexes too. if (table.getType() != PTableType.VIEW) { - List names = Lists.newArrayListWithExpectedSize(2); - final List physicalNames = Lists.newArrayListWithExpectedSize(2); if (table.isMultiTenant() || MetaDataUtil.hasViewIndexTable(connection, table.getPhysicalName())) { - names.add(PNameFactory.newName(SchemaUtil.getTableName( - MetaDataUtil.getViewIndexSchemaName(table.getSchemaName().getString()), - MetaDataUtil.getViewIndexTableName(table.getTableName().getString())))); - physicalNames.add(PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes()))); - } - if (MetaDataUtil.hasLocalIndexTable(connection, table.getPhysicalName())) { - names.add(PNameFactory.newName(SchemaUtil.getTableName( - MetaDataUtil.getLocalIndexSchemaName(table.getSchemaName().getString()), - MetaDataUtil.getLocalIndexTableName(table.getTableName().getString())))); - physicalNames.add(PNameFactory.newName(MetaDataUtil.getLocalIndexPhysicalName(table.getPhysicalName().getBytes()))); - } - int i = 0; - for (final PName name : names) { - final int index = i++; + final PName physicalName = PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes())); PTable indexLogicalTable = new DelegateTable(table) { @Override public PName getPhysicalName() { - return physicalNames.get(index); + return physicalName; } }; - rowCount += updateStatisticsInternal(name, indexLogicalTable, updateStatisticsStmt.getProps()); + rowCount += updateStatisticsInternal(physicalName, indexLogicalTable, updateStatisticsStmt.getProps()); + } + PName physicalName = table.getPhysicalName(); + List localCFs = MetaDataUtil.getLocalIndexColumnFamilies(connection, physicalName.getBytes()); + if (!localCFs.isEmpty()) { + rowCount += updateStatisticsInternal(physicalName, table, updateStatisticsStmt.getProps(), localCFs); } } } @@ -1030,6 +1022,10 @@ public class MetaDataClient { } private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map statsProps) throws SQLException { + return updateStatisticsInternal(physicalName, logicalTable, statsProps, null); + } + + private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map statsProps, List cfs) throws SQLException { ReadOnlyProps props = connection.getQueryServices().getProps(); final long msMinBetweenUpdates = props .getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB, @@ -1063,7 +1059,7 @@ public class MetaDataClient { } }; TableRef tableRef = new TableRef(null, nonTxnLogicalTable, clientTimeStamp, false); - MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp); + MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, cfs, null, clientTimeStamp); Scan scan = plan.getContext().getScan(); scan.setCacheBlocks(false); scan.setAttribute(ANALYZE_TABLE, TRUE_BYTES); @@ -1764,6 +1760,7 @@ public class MetaDataClient { updateCacheFrequency = updateCacheFrequencyProp; } String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps); + Long guidePostWidth = (Long) TableProperty.GUIDE_POSTS_WIDTH.getValue(tableProps); Boolean storeNullsProp = (Boolean) TableProperty.STORE_NULLS.getValue(tableProps); if (storeNullsProp == null) { @@ -2230,6 +2227,11 @@ public class MetaDataClient { tableUpsert.setString(23, autoPartitionSeq); } tableUpsert.setBoolean(24, isAppendOnlySchema); + if (guidePostWidth == null) { + tableUpsert.setNull(25, Types.BIGINT); + } else { + tableUpsert.setLong(25, guidePostWidth); + } tableUpsert.execute(); if (asyncCreatedDate != null) { @@ -2631,11 +2633,12 @@ public class MetaDataClient { } private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional, Long updateCacheFrequency) throws SQLException { - return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null); + return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null, -1L); } private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, - Boolean isTransactional, Long updateCacheFrequency, Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls) + Boolean isTransactional, Long updateCacheFrequency, Boolean isImmutableRows, Boolean disableWAL, + Boolean isMultiTenant, Boolean storeNulls, Long guidePostWidth) throws SQLException { String schemaName = table.getSchemaName().getString(); String tableName = table.getTableName().getString(); @@ -2673,6 +2676,9 @@ public class MetaDataClient { if (updateCacheFrequency != null) { mutateLongProperty(tenantId, schemaName, tableName, UPDATE_CACHE_FREQUENCY, updateCacheFrequency); } + if (guidePostWidth == null || guidePostWidth >= 0) { + mutateLongProperty(tenantId, schemaName, tableName, GUIDE_POSTS_WIDTH, guidePostWidth); + } return seqNum; } @@ -2694,7 +2700,7 @@ public class MetaDataClient { } private void mutateLongProperty(String tenantId, String schemaName, String tableName, - String propertyName, long propertyValue) throws SQLException { + String propertyName, Long propertyValue) throws SQLException { String updatePropertySql = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " + TENANT_ID + "," + TABLE_SCHEM + "," + @@ -2705,7 +2711,11 @@ public class MetaDataClient { tableBoolUpsert.setString(1, tenantId); tableBoolUpsert.setString(2, schemaName); tableBoolUpsert.setString(3, tableName); - tableBoolUpsert.setLong(4, propertyValue); + if (propertyValue == null) { + tableBoolUpsert.setNull(4, Types.BIGINT); + } else { + tableBoolUpsert.setLong(4, propertyValue); + } tableBoolUpsert.execute(); } } @@ -2733,6 +2743,7 @@ public class MetaDataClient { Boolean storeNullsProp = null; Boolean isTransactionalProp = null; Long updateCacheFrequencyProp = null; + Long guidePostWidth = -1L; Map>> properties = new HashMap<>(stmtProperties.size()); List columnDefs = null; @@ -2791,6 +2802,8 @@ public class MetaDataClient { isTransactionalProp = (Boolean)value; } else if (propName.equals(UPDATE_CACHE_FREQUENCY)) { updateCacheFrequencyProp = (Long)value; + } else if (propName.equals(GUIDE_POSTS_WIDTH)) { + guidePostWidth = (Long)value; } } // if removeTableProps is true only add the property if it is not a HTable or Phoenix Table property @@ -2857,6 +2870,9 @@ public class MetaDataClient { changingPhoenixTableProperty = true; } } + if (guidePostWidth == null || guidePostWidth >= 0) { + changingPhoenixTableProperty = true; + } Boolean storeNulls = null; if (storeNullsProp != null) { if (storeNullsProp.booleanValue() != table.getStoreNulls()) { @@ -2996,7 +3012,8 @@ public class MetaDataClient { } long seqNum = table.getSequenceNumber(); if (changingPhoenixTableProperty || columnDefs.size() > 0) { - seqNum = incrementTableSeqNum(table, tableType, columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows, disableWAL, multiTenant, storeNulls); + seqNum = incrementTableSeqNum(table, tableType, columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows, + disableWAL, multiTenant, storeNulls, guidePostWidth); tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond()); connection.rollback(); } @@ -3091,7 +3108,7 @@ public class MetaDataClient { connection.setAutoCommit(true); // Delete everything in the column. You'll still be able to do queries at earlier timestamps long ts = (scn == null ? result.getMutationTime() : scn); - MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, projectCF, null, ts); + MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, Collections.singletonList(projectCF), null, ts); return connection.getQueryServices().updateData(plan); } return new MutationState(0,connection); http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java index d5d0b84..26a7718 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java @@ -77,6 +77,13 @@ public enum TableProperty { }, APPEND_ONLY_SCHEMA(PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, true), + GUIDE_POSTS_WIDTH(PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH, true, false) { + @Override + public Object getValue(Object value) { + return value == null ? null : ((Number) value).longValue(); + } + + }, ; private final String propertyName; http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java index 3bb9898..0cf5ed8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java @@ -21,12 +21,17 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.regionserver.InternalScanner; @@ -35,11 +40,14 @@ import org.apache.hadoop.hbase.regionserver.Store; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.types.PInteger; import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.util.MetaDataUtil; +import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TimeKeeper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,24 +61,21 @@ class DefaultStatisticsCollector implements StatisticsCollector { private static final Logger logger = LoggerFactory.getLogger(DefaultStatisticsCollector.class); private final Map> guidePostsInfoWriterMap = Maps.newHashMap(); private final StatisticsWriter statsWriter; - private final Pair cachedGps; + private final Pair cachedGuidePosts; + private final byte[] guidePostWidthBytes; + private final byte[] guidePostPerRegionBytes; + // Where to look for GUIDE_POST_WIDTH in SYSTEM.CATALOG + private final byte[] ptableKey; + private final RegionCoprocessorEnvironment env; - private long guidepostDepth; + private long guidePostDepth; private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; DefaultStatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family, byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException { - Configuration config = env.getConfiguration(); - int guidepostPerRegion = gp_per_region_bytes == null - ? config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB, - QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION) - : PInteger.INSTANCE.getCodec().decodeInt(gp_per_region_bytes, 0, SortOrder.getDefault()); - long guidepostWidth = gp_width_bytes == null - ? config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, - QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES) - : PLong.INSTANCE.getCodec().decodeInt(gp_width_bytes, 0, SortOrder.getDefault()); - this.guidepostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth, - env.getRegion().getTableDesc()); + this.env = env; + this.guidePostWidthBytes = gp_width_bytes; + this.guidePostPerRegionBytes = gp_per_region_bytes; // Provides a means of clients controlling their timestamps to not use current time // when background tasks are updating stats. Instead we track the max timestamp of // the cells and use that. @@ -80,16 +85,64 @@ class DefaultStatisticsCollector implements StatisticsCollector { if (!useCurrentTime) { clientTimeStamp = DefaultStatisticsCollector.NO_TIMESTAMP; } + String pName = tableName; + // For view index, get GUIDE_POST_WIDTH from data physical table + // since there's no row representing those in SYSTEM.CATALOG. + if (MetaDataUtil.isViewIndex(tableName)) { + pName = MetaDataUtil.getViewIndexUserTableName(tableName); + } + ptableKey = SchemaUtil.getTableKeyFromFullName(pName); // Get the stats table associated with the current table on which the CP is // triggered this.statsWriter = StatisticsWriter.newWriter(env, tableName, clientTimeStamp); // in a compaction we know the one family ahead of time if (family != null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family); - cachedGps = new Pair(0l, new GuidePostsInfoBuilder()); - guidePostsInfoWriterMap.put(cfKey, cachedGps); + cachedGuidePosts = new Pair(0l, new GuidePostsInfoBuilder()); + guidePostsInfoWriterMap.put(cfKey, cachedGuidePosts); + } else { + cachedGuidePosts = null; + } + } + + private void initGuidepostDepth() throws IOException { + // First check is if guidepost info set on statement itself + if (guidePostPerRegionBytes != null || guidePostWidthBytes != null) { + int guidepostPerRegion = 0; + long guidepostWidth = QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES; + if (guidePostPerRegionBytes != null) { + guidepostPerRegion = PInteger.INSTANCE.getCodec().decodeInt(guidePostPerRegionBytes, 0, SortOrder.getDefault()); + } + if (guidePostWidthBytes != null) { + guidepostWidth = PLong.INSTANCE.getCodec().decodeInt(guidePostWidthBytes, 0, SortOrder.getDefault()); + } + this.guidePostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth, + env.getRegion().getTableDesc()); } else { - cachedGps = null; + // Next check for GUIDE_POST_WIDTH on table + HTableInterface htable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); + Get get = new Get(ptableKey); + get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES); + Result result = htable.get(get); + long guidepostWidth = 0; + if (!result.isEmpty()) { + Cell cell = result.listCells().get(0); + guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(), cell.getValueOffset(), SortOrder.getDefault()); + } + if (guidepostWidth > 0) { + this.guidePostDepth = guidepostWidth; + } else { + // Last use global config value + Configuration config = env.getConfiguration(); + this.guidePostDepth = StatisticsUtil.getGuidePostDepth( + config.getInt( + QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB, + QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION), + config.getLong( + QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, + QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES), + env.getRegion().getTableDesc()); + } } } @@ -114,28 +167,32 @@ class DefaultStatisticsCollector implements StatisticsCollector { commitStats(mutations); } catch (IOException e) { logger.error("Unable to commit new stats", e); - } finally { - clear(); } } private void writeStatsToStatsTable(final Region region, boolean delete, List mutations, long currentTime) throws IOException { try { + Set fams = guidePostsInfoWriterMap.keySet(); // update the statistics table // Delete statistics for a region if no guidepost is collected for that region during UPDATE STATISTICS // This will not impact a stats collection of single column family during compaction as // guidePostsInfoWriterMap cannot be empty in this case. - if (cachedGps == null) { + if (cachedGuidePosts == null) { + boolean collectingForLocalIndex = !fams.isEmpty() && MetaDataUtil.isLocalIndexFamily(fams.iterator().next()); for (Store store : region.getStores()) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName()); + boolean isLocalIndexStore = MetaDataUtil.isLocalIndexFamily(cfKey); + if (isLocalIndexStore != collectingForLocalIndex) { + continue; + } if (!guidePostsInfoWriterMap.containsKey(cfKey)) { Pair emptyGps = new Pair(0l, new GuidePostsInfoBuilder()); guidePostsInfoWriterMap.put(cfKey, emptyGps); } } } - for (ImmutableBytesPtr fam : guidePostsInfoWriterMap.keySet()) { + for (ImmutableBytesPtr fam : fams) { if (delete) { if (logger.isDebugEnabled()) { logger.debug("Deleting the stats for the region " + region.getRegionInfo()); @@ -162,6 +219,7 @@ class DefaultStatisticsCollector implements StatisticsCollector { * * @param results * next batch of {@link KeyValue}s + * @throws IOException */ @Override public void collectStatistics(final List results) { @@ -171,7 +229,7 @@ class DefaultStatisticsCollector implements StatisticsCollector { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp()); Pair gps; - if (cachedGps == null) { + if (cachedGuidePosts == null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()); gps = guidePostsInfoWriterMap.get(cfKey); @@ -185,16 +243,16 @@ class DefaultStatisticsCollector implements StatisticsCollector { gps.getSecond().incrementRowCount(); } } else { - gps = cachedGps; + gps = cachedGuidePosts; if (incrementRow) { - cachedGps.getSecond().incrementRowCount(); + cachedGuidePosts.getSecond().incrementRowCount(); incrementRow = false; } } int kvLength = kv.getLength(); long byteCount = gps.getFirst() + kvLength; gps.setFirst(byteCount); - if (byteCount >= guidepostDepth) { + if (byteCount >= guidePostDepth) { ImmutableBytesWritable row = new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()); if (gps.getSecond().addGuidePosts(row, byteCount, gps.getSecond().getRowCount())) { gps.setFirst(0l); @@ -221,9 +279,10 @@ class DefaultStatisticsCollector implements StatisticsCollector { } @Override - public void clear() { + public void init() throws IOException { this.guidePostsInfoWriterMap.clear(); maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP; + initGuidepostDepth(); } @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java index 1063229..a16a48a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java @@ -21,13 +21,10 @@ import java.io.IOException; import java.util.List; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.Store; -import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; /** @@ -62,7 +59,8 @@ public class NoOpStatisticsCollector implements StatisticsCollector { return delegate; } - @Override public void clear() { + @Override + public void init() { // No-op } http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java index 1dcab08..43ec6c7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java @@ -23,7 +23,6 @@ import java.util.List; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; -import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.Store; @@ -50,6 +49,7 @@ public interface StatisticsCollector extends Closeable { /** * Collect statistics for the given list of cells. This method can be called multiple times * during collection of statistics. + * @throws IOException */ void collectStatistics(List results); @@ -60,9 +60,10 @@ public interface StatisticsCollector extends Closeable { InternalScanner delegate) throws IOException; /** - * Clear all statistics information that has been collected. + * Called before beginning the collection of statistics through {@link #collectStatistics(List)} + * @throws IOException */ - void clear(); + void init() throws IOException; /** * Retrieve the calculated guide post info for the given column family. http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java index bb21f1b..bbc7cd4 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java @@ -29,27 +29,28 @@ import org.apache.phoenix.query.QueryServices; public class StatisticsCollectorFactory { public static StatisticsCollector createStatisticsCollector(RegionCoprocessorEnvironment env, - String tableName, long clientTimestamp, byte[] guidepostWidthBytes, + String tableName, long clientTimeStamp, byte[] guidepostWidthBytes, byte[] guidepostsPerRegionBytes) throws IOException { - if (statisticsEnabled(env)) { - return new DefaultStatisticsCollector(env, tableName, clientTimestamp, null, - guidepostWidthBytes, guidepostsPerRegionBytes); - } else { - return new NoOpStatisticsCollector(); - } + return createStatisticsCollector(env, tableName, clientTimeStamp, null, guidepostWidthBytes, guidepostsPerRegionBytes); } public static StatisticsCollector createStatisticsCollector( RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] storeName) throws IOException { + return createStatisticsCollector(env, tableName, clientTimeStamp, storeName, null, null); + } + + public static StatisticsCollector createStatisticsCollector( + RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, + byte[] storeName, byte[] guidepostWidthBytes, + byte[] guidepostsPerRegionBytes) throws IOException { if (statisticsEnabled(env)) { return new DefaultStatisticsCollector(env, tableName, clientTimeStamp, storeName, - null, null); + guidepostWidthBytes, guidepostsPerRegionBytes); } else { return new NoOpStatisticsCollector(); } } - /** * Determines if statistics are enabled (which is the default). This is done on the http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java index 736efc6..8306940 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java @@ -83,8 +83,9 @@ public class StatisticsScanner implements InternalScanner { * * @param results * next batch of {@link KeyValue}s + * @throws IOException */ - private void updateStats(final List results) { + private void updateStats(final List results) throws IOException { if (!results.isEmpty()) { tracker.collectStatistics(results); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java index db31b69..685c24e 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java @@ -20,7 +20,6 @@ import static org.apache.phoenix.util.SchemaUtil.getVarCharLength; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.SortedMap; @@ -30,7 +29,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Result; @@ -99,7 +97,7 @@ public class StatisticsUtil { return rowKey; } - public static byte[] getKey(byte[] table, ImmutableBytesPtr fam) { + private static byte[] getKey(byte[] table, ImmutableBytesPtr fam) { // always starts with the source table and column family byte[] rowKey = new byte[table.length + fam.getLength() + 1]; int offset = 0; @@ -111,11 +109,7 @@ public class StatisticsUtil { return rowKey; } - public static byte[] copyRow(KeyValue kv) { - return Arrays.copyOfRange(kv.getRowArray(), kv.getRowOffset(), kv.getRowOffset() + kv.getRowLength()); - } - - public static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesPtr cf, boolean nextKey) { + private static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesPtr cf, boolean nextKey) { if (Bytes.compareTo(key, ByteUtil.EMPTY_BYTE_ARRAY) != 0) { return getRowKey(tableNameBytes, cf, key); } key = ByteUtil.concat(getKey(tableNameBytes, cf), QueryConstants.SEPARATOR_BYTE_ARRAY); if (nextKey) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java index 9089b68..9622880 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java @@ -491,7 +491,7 @@ public class IndexUtil { joinResult = dataRegion.get(get); } else { TableName dataTable = - TableName.valueOf(MetaDataUtil.getUserTableName(c.getEnvironment() + TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(c.getEnvironment() .getRegion().getTableDesc().getNameAsString())); HTableInterface table = null; try { http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java index c72e404..52346a5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -214,7 +215,7 @@ public class MetaDataUtil { } public static long getSequenceNumber(List tableMetaData) { - return getSequenceNumber(getTableHeaderRow(tableMetaData)); + return getSequenceNumber(getPutOnlyTableHeaderRow(tableMetaData)); } public static PTableType getTableType(List tableMetaData, KeyValueBuilder builder, @@ -366,7 +367,7 @@ public class MetaDataUtil { return schemaName; } - public static String getUserTableName(String localIndexTableName) { + public static String getLocalIndexUserTableName(String localIndexTableName) { if (localIndexTableName.contains(QueryConstants.NAMESPACE_SEPARATOR)) { String schemaName = SchemaUtil.getSchemaNameFromFullName(localIndexTableName, QueryConstants.NAMESPACE_SEPARATOR); @@ -384,6 +385,24 @@ public class MetaDataUtil { } } + public static String getViewIndexUserTableName(String viewIndexTableName) { + if (viewIndexTableName.contains(QueryConstants.NAMESPACE_SEPARATOR)) { + String schemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexTableName, + QueryConstants.NAMESPACE_SEPARATOR); + String tableName = SchemaUtil.getTableNameFromFullName(viewIndexTableName, + QueryConstants.NAMESPACE_SEPARATOR); + String userTableName = tableName.substring(VIEW_INDEX_TABLE_PREFIX.length()); + return (schemaName + QueryConstants.NAMESPACE_SEPARATOR + userTableName); + } else { + String schemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexTableName); + if (!schemaName.isEmpty()) schemaName = schemaName.substring(VIEW_INDEX_TABLE_PREFIX.length()); + String tableName = viewIndexTableName.substring( + (schemaName.isEmpty() ? 0 : (schemaName.length() + QueryConstants.NAME_SEPARATOR.length())) + + VIEW_INDEX_TABLE_PREFIX.length()); + return SchemaUtil.getTableName(schemaName, tableName); + } + } + public static String getViewIndexSequenceSchemaName(PName physicalName, boolean isNamespaceMapped) { if (!isNamespaceMapped) { return VIEW_INDEX_SEQUENCE_PREFIX + physicalName.getString(); } return SchemaUtil.getSchemaNameFromFullName(physicalName.toString()); @@ -461,6 +480,18 @@ public class MetaDataUtil { return families; } + public static List getLocalIndexColumnFamilies(PhoenixConnection conn, byte[] physicalTableName) throws SQLException { + HTableDescriptor desc = conn.getQueryServices().getTableDescriptor(physicalTableName); + if(desc == null ) return Collections.emptyList(); + List families = new ArrayList(desc.getColumnFamilies().length / 2); + for (HColumnDescriptor cf : desc.getColumnFamilies()) { + if (cf.getNameAsString().startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX)) { + families.add(cf.getName()); + } + } + return families; + } + public static void deleteViewIndexSequences(PhoenixConnection connection, PName name, boolean isNamespaceMapped) throws SQLException { String schemaName = getViewIndexSequenceSchemaName(name, isNamespaceMapped); @@ -603,4 +634,9 @@ public class MetaDataUtil { public static boolean isHTableProperty(String propName) { return !isHColumnProperty(propName) && !TableProperty.isPhoenixTableProperty(propName); } + + public static boolean isLocalIndexFamily(ImmutableBytesPtr cfPtr) { + return cfPtr.getLength() >= QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length && + Bytes.compareTo(cfPtr.get(), cfPtr.getOffset(), QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES, 0, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length) == 0; + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java index cddebb7..93b3f73 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java @@ -348,7 +348,7 @@ public class UpgradeUtil { HTableDescriptor[] localIndexTables = admin.listTables(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX+".*"); String localIndexSplitter = LocalIndexSplitter.class.getName(); for (HTableDescriptor table : localIndexTables) { - HTableDescriptor dataTableDesc = admin.getTableDescriptor(TableName.valueOf(MetaDataUtil.getUserTableName(table.getNameAsString()))); + HTableDescriptor dataTableDesc = admin.getTableDescriptor(TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString()))); HColumnDescriptor[] columnFamilies = dataTableDesc.getColumnFamilies(); boolean modifyTable = false; for(HColumnDescriptor cf : columnFamilies) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java index c91b855..7697d8c 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java @@ -2257,6 +2257,22 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest { } } + @Test + public void testNegativeGuidePostWidth() throws Exception { + Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES); + try (Connection conn = DriverManager.getConnection(getUrl(), props);) { + try { + conn.createStatement().execute( + "CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR) GUIDE_POST_WIDTH = -1"); + fail(); + } catch (SQLException e) { + assertEquals("Unexpected Exception", + SQLExceptionCode.PARSER_ERROR + .getErrorCode(), e.getErrorCode()); + } + } + } + private static void assertFamilies(Scan s, String... families) { assertEquals(families.length, s.getFamilyMap().size()); for (String fam : families) {