phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jamestay...@apache.org
Subject [1/3] phoenix git commit: PHOENIX-3953 Clear INDEX_DISABLED_TIMESTAMP and disable index on compaction
Date Fri, 25 Aug 2017 20:27:39 GMT
Repository: phoenix
Updated Branches:
  refs/heads/master a5fb51948 -> 435441ea8


PHOENIX-3953 Clear INDEX_DISABLED_TIMESTAMP and disable index on compaction


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/435441ea
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/435441ea
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/435441ea

Branch: refs/heads/master
Commit: 435441ea8ba336e1967b03cf84f1868c5ef14790
Parents: 7d3368b
Author: James Taylor <jtaylor@salesforce.com>
Authored: Fri Aug 25 11:38:41 2017 -0700
Committer: James Taylor <jtaylor@salesforce.com>
Committed: Fri Aug 25 13:27:23 2017 -0700

----------------------------------------------------------------------
 .../end2end/index/PartialIndexRebuilderIT.java  | 32 ++++++++++++++
 .../org/apache/phoenix/hbase/index/Indexer.java | 45 ++++++++++++++++++++
 .../java/org/apache/phoenix/util/IndexUtil.java |  7 ++-
 3 files changed, 83 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/435441ea/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
index f42d2c9..9483e87 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialIndexRebuilderIT.java
@@ -245,6 +245,38 @@ public class PartialIndexRebuilderIT extends BaseUniqueNamesOwnClusterIT
{
     }
     
     @Test
+    public void testCompactionDuringRebuild() throws Throwable {
+        String schemaName = generateUniqueName();
+        String tableName = generateUniqueName();
+        String indexName = generateUniqueName();
+        final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+        String fullIndexName = SchemaUtil.getTableName(schemaName, indexName);
+        final MyClock clock = new MyClock(1000);
+        // Use our own clock to prevent race between partial rebuilder and compaction
+        EnvironmentEdgeManager.injectEdge(clock);
+        try (Connection conn = DriverManager.getConnection(getUrl())) {
+            conn.createStatement().execute("CREATE TABLE " + fullTableName + "(k INTEGER
PRIMARY KEY, v1 INTEGER, v2 INTEGER) COLUMN_ENCODED_BYTES = 0, STORE_NULLS=true, GUIDE_POSTS_WIDTH=1000");
+            clock.time += 1000;
+            conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + fullTableName
+ " (v1) INCLUDE (v2)");
+            clock.time += 1000;
+            conn.createStatement().execute("UPSERT INTO " + fullTableName + " VALUES(1, 2,
3)");
+            conn.commit();
+            clock.time += 1000;
+            long disableTS = EnvironmentEdgeManager.currentTimeMillis();
+            HTableInterface metaTable = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES);
+            IndexUtil.updateIndexState(fullIndexName, disableTS, metaTable, PIndexState.DISABLE);
+            TestUtil.doMajorCompaction(conn, fullIndexName);
+            assertFalse(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.DISABLE,
0L));
+            TestUtil.analyzeTable(conn, fullTableName);
+            assertFalse(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.DISABLE,
0L));
+            TestUtil.doMajorCompaction(conn, fullTableName);
+            assertTrue(TestUtil.checkIndexState(conn, fullIndexName, PIndexState.DISABLE,
0L));
+        } finally {
+            EnvironmentEdgeManager.injectEdge(null);
+        }
+    }
+
+    @Test
     @Repeat(5)
     public void testDeleteAndUpsertAfterFailure() throws Throwable {
         final int nRows = 10;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/435441ea/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
index ba8f3d4..b76752d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/Indexer.java
@@ -25,6 +25,7 @@ import static org.apache.phoenix.hbase.index.write.IndexWriterUtils.INDEX_WRITER
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -87,12 +88,18 @@ import org.apache.phoenix.hbase.index.write.IndexWriter;
 import org.apache.phoenix.hbase.index.write.RecoveryIndexWriter;
 import org.apache.phoenix.hbase.index.write.recovery.PerRegionIndexWriteCache;
 import org.apache.phoenix.hbase.index.write.recovery.StoreFailuresInCachePolicy;
+import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PIndexState;
+import org.apache.phoenix.schema.PTable;
 import org.apache.phoenix.trace.TracingUtils;
 import org.apache.phoenix.trace.util.NullSpan;
 import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.apache.phoenix.util.IndexUtil;
 import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
 import org.apache.phoenix.util.ServerUtil;
 
 import com.google.common.collect.Lists;
@@ -829,5 +836,43 @@ public class Indexer extends BaseRegionObserver {
     properties.put(Indexer.INDEX_BUILDER_CONF_KEY, builder.getName());
     desc.addCoprocessor(Indexer.class.getName(), null, priority, properties);
   }
+  
+  @Override
+  public InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment>
c, final Store store,
+          final InternalScanner scanner, final ScanType scanType) throws IOException {
+      // Compaction and split upcalls run with the effective user context of the requesting
user.
+      // This will lead to failure of cross cluster RPC if the effective user is not
+      // the login user. Switch to the login user context to ensure we have the expected
+      // security context.
+      return User.runAsLoginUser(new PrivilegedExceptionAction<InternalScanner>() {
+          @Override
+          public InternalScanner run() throws Exception {
+              InternalScanner internalScanner = scanner;
+              if (scanType.equals(ScanType.COMPACT_DROP_DELETES)) {
+                  String fullTableName = c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
+                  try {
+                      PhoenixConnection conn =  QueryUtil.getConnectionOnServer(c.getEnvironment().getConfiguration()).unwrap(PhoenixConnection.class);
+                      PTable table = PhoenixRuntime.getTableNoCache(conn, fullTableName);
+                      for (PTable index : table.getIndexes()) {
+                          if (index.getIndexDisableTimestamp() != 0) {
+                              try {
+                                  IndexUtil.updateIndexState(conn, index.getName().getString(),
PIndexState.DISABLE, Long.valueOf(0L));
+                              } catch (SQLException e) {
+                                  LOG.warn("Unable to permanently disable index " + index.getName().getString(),
e);
+                              }
+                          }
+                      }
+                  } catch (Exception e) {
+                      // If we can't reach the stats table, don't interrupt the normal
+                    // compaction operation, just log a warning.
+                    if (LOG.isWarnEnabled()) {
+                        LOG.warn("Unable to permanently disable indexes being partially rebuild
for " + fullTableName, e);
+                    }
+                  }
+              }
+              return internalScanner;
+          }
+      });
+  }
 }
 

http://git-wip-us.apache.org/repos/asf/phoenix/blob/435441ea/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 75a7e55..ea6fb96 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -711,10 +711,15 @@ public class IndexUtil {
             mutations.toArray(new Mutation[mutations.size()]),
             HConstants.NO_NONCE, HConstants.NO_NONCE);
     }
-
+    
     public static MetaDataMutationResult updateIndexState(String indexTableName, long minTimeStamp,
             HTableInterface metaTable, PIndexState newState) throws Throwable {
         byte[] indexTableKey = SchemaUtil.getTableKeyFromFullName(indexTableName);
+        return updateIndexState(indexTableKey, minTimeStamp, metaTable, newState);
+    }
+    
+    public static MetaDataMutationResult updateIndexState(byte[] indexTableKey, long minTimeStamp,
+            HTableInterface metaTable, PIndexState newState) throws Throwable {
         // Mimic the Put that gets generated by the client on an update of the index state
         Put put = new Put(indexTableKey);
         put.add(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.INDEX_STATE_BYTES,


Mime
View raw message