phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jamestay...@apache.org
Subject phoenix git commit: PHOENIX-3953 Clear INDEX_DISABLED_TIMESTAMP and disable index on compaction (addendum)
Date Tue, 05 Sep 2017 21:24:19 GMT
Repository: phoenix
Updated Branches:
  refs/heads/4.x-HBase-1.1 d82e55168 -> 131abf8ff


PHOENIX-3953 Clear INDEX_DISABLED_TIMESTAMP and disable index on compaction (addendum)


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/131abf8f
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/131abf8f
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/131abf8f

Branch: refs/heads/4.x-HBase-1.1
Commit: 131abf8ff786acd8a8895555303b99aafbff1205
Parents: d82e551
Author: James Taylor <jtaylor@salesforce.com>
Authored: Tue Sep 5 13:05:17 2017 -0700
Committer: James Taylor <jtaylor@salesforce.com>
Committed: Tue Sep 5 14:23:18 2017 -0700

----------------------------------------------------------------------
 .../end2end/index/PartialIndexRebuilderIT.java  | 21 +++--
 .../UngroupedAggregateRegionObserver.java       |  2 +-
 .../stats/DefaultStatisticsCollector.java       | 83 ++++++++++++++------
 .../schema/stats/NoOpStatisticsCollector.java   |  2 +-
 .../schema/stats/StatisticsCollector.java       |  2 +-
 5 files changed, 75 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/131abf8f/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
index 9483e87..139725f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
@@ -248,29 +248,34 @@ public class PartialIndexRebuilderIT extends BaseUniqueNamesOwnClusterIT
{
     public void testCompactionDuringRebuild() throws Throwable {
         String schemaName = generateUniqueName();
         String tableName = generateUniqueName();
-        String indexName = generateUniqueName();
+        String indexName1 = generateUniqueName();
+        String indexName2 = generateUniqueName();
         final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
-        String fullIndexName = SchemaUtil.getTableName(schemaName, indexName);
+        String fullIndexName1 = SchemaUtil.getTableName(schemaName, indexName1);
+        String fullIndexName2 = SchemaUtil.getTableName(schemaName, indexName2);
         final MyClock clock = new MyClock(1000);
         // Use our own clock to prevent race between partial rebuilder and compaction
         EnvironmentEdgeManager.injectEdge(clock);
         try (Connection conn = DriverManager.getConnection(getUrl())) {
             conn.createStatement().execute("CREATE TABLE " + fullTableName + "(k INTEGER
PRIMARY KEY, v1 INTEGER, v2 INTEGER) COLUMN_ENCODED_BYTES = 0, STORE_NULLS=true, GUIDE_POSTS_WIDTH=1000");
             clock.time += 1000;
-            conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + fullTableName
+ " (v1) INCLUDE (v2)");
+            conn.createStatement().execute("CREATE INDEX " + indexName1 + " ON " + fullTableName
+ " (v1) INCLUDE (v2)");
+            clock.time += 1000;
+            conn.createStatement().execute("CREATE INDEX " + indexName2 + " ON " + fullTableName
+ " (v2) INCLUDE (v1)");
             clock.time += 1000;
             conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(1, 2,
3)");
             conn.commit();
             clock.time += 1000;
             long disableTS = EnvironmentEdgeManager.currentTimeMillis();
             HTableInterface metaTable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES);
-            IndexUtil.updateIndexState(fullIndexName, disableTS, metaTable, PIndexState.DISABLE);
-            TestUtil.doMajorCompaction(conn, fullIndexName);
-            assertFalse(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.DISABLE,
0L));
+            IndexUtil.updateIndexState(fullIndexName1, disableTS, metaTable, PIndexState.DISABLE);
+            IndexUtil.updateIndexState(fullIndexName2, disableTS, metaTable, PIndexState.DISABLE);
+            TestUtil.doMajorCompaction(conn, fullIndexName1);
+            assertTrue(TestUtil.checkIndexState(conn, fullIndexName1, PIndexState.DISABLE,
0L));
             TestUtil.analyzeTable(conn, fullTableName);
-            assertFalse(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.DISABLE,
0L));
+            assertFalse(TestUtil.checkIndexState(conn, fullIndexName2, PIndexState.DISABLE,
0L));
             TestUtil.doMajorCompaction(conn, fullTableName);
-            assertTrue(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.DISABLE,
0L));
+            assertTrue(TestUtil.checkIndexState(conn, fullIndexName2, PIndexState.DISABLE,
0L));
         } finally {
             EnvironmentEdgeManager.injectEdge(null);
         }

http://git-wip-us.apache.org/repos/asf/phoenix/blob/131abf8f/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index 31b8e36..31c83e4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -1115,7 +1115,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
             long rowCount = 0;
             try {
                 if (!compactionRunning) {
-                    stats.init();
+                    stats.init(false);
                     synchronized (innerScanner) {
                         do {
                             List<Cell> results = new ArrayList<Cell>();

http://git-wip-us.apache.org/repos/asf/phoenix/blob/131abf8f/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
index b8ba759..61a6fa2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTableInterface;
 import org.apache.hadoop.hbase.client.Mutation;
@@ -41,13 +42,16 @@ import org.apache.hadoop.hbase.regionserver.Region;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.phoenix.coprocessor.MetaDataProtocol;
+import org.apache.phoenix.coprocessor.MetaDataProtocol.MutationCode;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PIndexState;
 import org.apache.phoenix.schema.SortOrder;
 import org.apache.phoenix.schema.types.PInteger;
 import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.MetaDataUtil;
 import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.util.TimeKeeper;
@@ -109,9 +113,10 @@ class DefaultStatisticsCollector implements StatisticsCollector {
         }
     }
     
-    private void initGuidepostDepth() throws IOException {
+    private void initGuidepostDepth(boolean isMajorCompaction) throws IOException {
         // First check is if guidepost info set on statement itself
-        if (guidePostPerRegionBytes != null || guidePostWidthBytes != null) {
+        boolean guidepostOnStatement = guidePostPerRegionBytes != null || guidePostWidthBytes
!= null;
+        if (guidepostOnStatement) {
             int guidepostPerRegion = 0;
             long guidepostWidth = QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES;
             if (guidePostPerRegionBytes != null) {
@@ -122,20 +127,48 @@ class DefaultStatisticsCollector implements StatisticsCollector {
             }
             this.guidePostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth,
                     env.getRegion().getTableDesc());
-        } else {
+        }
+        
+        if (!guidepostOnStatement || isMajorCompaction) {
             long guidepostWidth = -1;
             HTableInterface htable = null;
             try {
-                // Next check for GUIDE_POST_WIDTH on table
-                htable = env.getTable(
-                        SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES,
env.getConfiguration()));
+                // Next check for GUIDE_POST_WIDTH and INDEX_DISABLE_TIMESTAMP on table
+                TableName htableName = SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES,
env.getConfiguration());
+                htable = env.getTable(htableName);
                 Get get = new Get(ptableKey);
                 get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES);
+                get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES);
                 Result result = htable.get(get);
                 if (!result.isEmpty()) {
-                    Cell cell = result.listCells().get(0);
-                    guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(),
cell.getValueOffset(), SortOrder.getDefault());
+                    Cell gpwCell = result.getColumnLatestCell(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES);
+                    if (gpwCell != null) {
+                        guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(gpwCell.getValueArray(),
gpwCell.getValueOffset(), SortOrder.getDefault());
+                    }
+                    if (isMajorCompaction) {
+                        Cell idtsCell = result.getColumnLatestCell(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP_BYTES);
+                        if (idtsCell != null) {
+                            long indexDisableTimestamp = PLong.INSTANCE.getCodec().decodeLong(idtsCell.getValueArray(),
idtsCell.getValueOffset(), SortOrder.getDefault());
+                            // If we have a non zero value for INDEX_DISABLE_TIMESTAMP, that
means that our global mutable
+                            // secondary index needs to be partially rebuilt. If we're  compacting,
though, we may cleanup
+                            // the delete markers of an index *before* the puts for the same
row occur during replay. At
+                            // this point the partially index rebuild would leave the index
out of sync with the data
+                            // table. In that case, it's better to just permanently disable
the index and force it to be
+                            // manually rebuilt
+                            if (indexDisableTimestamp != 0) {
+                                MutationCode mutationCode = IndexUtil.updateIndexState(ptableKey,
0L, htable, PIndexState.DISABLE).getMutationCode();
+                                if (mutationCode != MutationCode.TABLE_ALREADY_EXISTS &&
mutationCode != MutationCode.TABLE_NOT_FOUND) {
+                                    LOG.warn("Attempt to permanently disable index " + env.getRegionInfo().getTable().getNameAsString()
+ 
+                                            " during compaction failed with code = " + mutationCode);
+                                }
+                            }
+                        }
+                    }
                 }
+            } catch (IOException e) {
+                throw e;
+            } catch (Throwable t) {
+                throw new IOException(t);
             } finally {
                 if (htable != null) {
                     try {
@@ -145,19 +178,21 @@ class DefaultStatisticsCollector implements StatisticsCollector {
                     }
                 }
             }
-            if (guidepostWidth >= 0) {
-                this.guidePostDepth = guidepostWidth;
-            } else {
-                // Last use global config value
-                Configuration config = env.getConfiguration();
-                this.guidePostDepth = StatisticsUtil.getGuidePostDepth(
-                        config.getInt(
-                            QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
-                            QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION),
-                        config.getLong(
-                            QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
-                            QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES),
-                        env.getRegion().getTableDesc());
+            if (!guidepostOnStatement) {
+                if (guidepostWidth >= 0) {
+                    this.guidePostDepth = guidepostWidth;
+                } else {
+                    // Last use global config value
+                    Configuration config = env.getConfiguration();
+                    this.guidePostDepth = StatisticsUtil.getGuidePostDepth(
+                            config.getInt(
+                                QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
+                                QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION),
+                            config.getLong(
+                                QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
+                                QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES),
+                            env.getRegion().getTableDesc());
+                }
             }
         }
     }
@@ -315,13 +350,13 @@ class DefaultStatisticsCollector implements StatisticsCollector {
         StatisticsScanner scanner = new StatisticsScanner(this, statsWriter, env, internalScan,
family);
         // We need to initialize the scanner synchronously and potentially perform a cross
region Get
         // in order to use the correct guide posts width for the table being compacted.
-        init();
+        init(true);
         return scanner;
     }
 
     @Override
-    public void init() throws IOException {
-        initGuidepostDepth();
+    public void init(boolean isMajorCompaction) throws IOException {
+        initGuidepostDepth(isMajorCompaction);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/phoenix/blob/131abf8f/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
index 74d1710..a13a722 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
@@ -61,7 +61,7 @@ public class NoOpStatisticsCollector implements StatisticsCollector {
     }
 
     @Override 
-    public void init() {
+    public void init(boolean isMajorCompaction) {
         // No-op
     }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/131abf8f/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index 60e83a8..9550469 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -64,7 +64,7 @@ public interface StatisticsCollector extends Closeable {
      * Called before beginning the collection of statistics through {@link #collectStatistics(List)}
      * @throws IOException 
      */
-    void init() throws IOException;
+    void init(boolean isMajorCompaction) throws IOException;
 
     /**
      * Retrieve the calculated guide post info for the given column family.


Mime
View raw message