phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tdsi...@apache.org
Subject [1/3] phoenix git commit: PHOENIX-2059 MR index build does not handle table with a schema name correctly
Date Wed, 01 Jul 2015 07:46:10 GMT
Repository: phoenix
Updated Branches:
  refs/heads/4.4-HBase-1.0 42a83e2e8 -> 3ff71e974


PHOENIX-2059 MR index build does not handle table with a schema name correctly


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3ff71e97
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3ff71e97
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3ff71e97

Branch: refs/heads/4.4-HBase-1.0
Commit: 3ff71e9740b53c14ca629c73fea40a1cb9fad6eb
Parents: 6dbbea9
Author: Thomas D'Silva <tdsilva@salesforce.com>
Authored: Mon Jun 22 17:45:58 2015 -0700
Committer: Thomas D'Silva <tdsilva@salesforce.com>
Committed: Wed Jul 1 00:45:56 2015 -0700

----------------------------------------------------------------------
 .../apache/phoenix/mapreduce/IndexToolIT.java   | 47 ++++++++++++--------
 .../phoenix/mapreduce/index/IndexTool.java      | 15 ++++---
 2 files changed, 36 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/3ff71e97/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/IndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/IndexToolIT.java
index 6761275..5d11cf2 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/mapreduce/IndexToolIT.java
@@ -42,6 +42,7 @@ import org.apache.phoenix.util.MetaDataUtil;
 import org.apache.phoenix.util.PhoenixRuntime;
 import org.apache.phoenix.util.PropertiesUtil;
 import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.SchemaUtil;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -62,6 +63,7 @@ public class IndexToolIT {
     public static void setUp() throws Exception {
         hbaseTestUtil = new HBaseTestingUtility();
         Configuration conf = hbaseTestUtil.getConfiguration();
+        conf.setBoolean("hbase.defaults.for.version.skip", true);
         setUpConfigForMiniCluster(conf);
         hbaseTestUtil.startMiniCluster();
         hbaseTestUtil.startMiniMapReduceCluster();
@@ -71,34 +73,35 @@ public class IndexToolIT {
     
     @Test
     public void testImmutableGlobalIndex() throws Exception {
-        testSecondaryIndex("DATA_TABLE1",true, false);
+        testSecondaryIndex("SCHEMA", "DATA_TABLE1", true, false);
     }
     
     @Test
     public void testImmutableLocalIndex() throws Exception {
-        testSecondaryIndex("DATA_TABLE2",true, true);
+        testSecondaryIndex("SCHEMA", "DATA_TABLE2", true, true);
     }
     
     @Test
     public void testMutableGlobalIndex() throws Exception {
-        testSecondaryIndex("DATA_TABLE3",false, false);
+        testSecondaryIndex("SCHEMA", "DATA_TABLE3", false, false);
     }
     
     @Test
     public void testMutableLocalIndex() throws Exception {
-        testSecondaryIndex("DATA_TABLE4",false, true);
+        testSecondaryIndex("SCHEMA", "DATA_TABLE4", false, true);
     }
     
-    public void testSecondaryIndex(final String dataTable , final boolean isImmutable , final
boolean isLocal) throws Exception {
+    public void testSecondaryIndex(final String schemaName, final String dataTable, final
boolean isImmutable , final boolean isLocal) throws Exception {
         
+    	final String fullTableName = SchemaUtil.getTableName(schemaName, dataTable);
         final String indxTable = String.format("%s_%s",dataTable,"INDX");
         Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
         Connection conn = DriverManager.getConnection(PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR
+ zkQuorum,props);
         Statement stmt = conn.createStatement();
         try {
         
-            stmt.execute(String.format("CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY KEY,
NAME VARCHAR, ZIP INTEGER) %s",dataTable, (isImmutable ? "IMMUTABLE_ROWS=true" :"")));
-            String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)",dataTable);
+            stmt.execute(String.format("CREATE TABLE %s (ID INTEGER NOT NULL PRIMARY KEY,
NAME VARCHAR, ZIP INTEGER) %s", fullTableName, (isImmutable ? "IMMUTABLE_ROWS=true" :"")));
+            String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", fullTableName);
             PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
             
             int id = 1;
@@ -107,15 +110,15 @@ public class IndexToolIT {
             upsertRow(stmt1, id++);
             conn.commit();
             
-            stmt.execute(String.format("CREATE %s INDEX %s ON %s (UPPER(NAME)) ASYNC ", (isLocal
? "LOCAL" : ""), indxTable,dataTable));
+            stmt.execute(String.format("CREATE %s INDEX %s ON %s  (UPPER(NAME)) ASYNC ",
(isLocal ? "LOCAL" : ""), indxTable, fullTableName));
    
             //verify rows are fetched from data table.
-            String selectSql = String.format("SELECT UPPER(NAME),ID FROM %s",dataTable);
+            String selectSql = String.format("SELECT UPPER(NAME),ID FROM %s", fullTableName);
             ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql);
             String actualExplainPlan = QueryUtil.getExplainPlan(rs);
             
             //assert we are pulling from data table.
-            assertEquals(String.format("CLIENT 1-CHUNK PARALLEL 1-WAY FULL SCAN OVER %s",dataTable),actualExplainPlan);
+            assertEquals(String.format("CLIENT 1-CHUNK PARALLEL 1-WAY FULL SCAN OVER %s",
fullTableName), actualExplainPlan);
             
             rs = stmt1.executeQuery(selectSql);
             assertTrue(rs.next());
@@ -127,7 +130,7 @@ public class IndexToolIT {
             final IndexTool indexingTool = new IndexTool();
             indexingTool.setConf(new Configuration(hbaseTestUtil.getConfiguration()));
             
-            final String[] cmdArgs = getArgValues(dataTable,indxTable);
+            final String[] cmdArgs = getArgValues(schemaName, dataTable, indxTable);
             int status = indexingTool.run(cmdArgs);
             assertEquals(0, status);
             
@@ -135,11 +138,13 @@ public class IndexToolIT {
             upsertRow(stmt1, 3);
             upsertRow(stmt1, 4);
             conn.commit();
+            
+            rs = stmt1.executeQuery("SELECT * FROM "+SchemaUtil.getTableName(schemaName,
indxTable));
 
             //assert we are pulling from index table.
             rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql);
             actualExplainPlan = QueryUtil.getExplainPlan(rs);
-            assertExplainPlan(actualExplainPlan,dataTable,indxTable,isLocal);
+            assertExplainPlan(actualExplainPlan,schemaName,dataTable,indxTable,isLocal);
             
             rs = stmt.executeQuery(selectSql);
             assertTrue(rs.next());
@@ -160,7 +165,7 @@ public class IndexToolIT {
       
             assertFalse(rs.next());
             
-            conn.createStatement().execute(String.format("DROP INDEX  %s ON %s",indxTable
, dataTable));
+            conn.createStatement().execute(String.format("DROP INDEX  %s ON %s",indxTable
, fullTableName));
         } finally {
             conn.close();
         }
@@ -219,14 +224,14 @@ public class IndexToolIT {
             final IndexTool indexingTool = new IndexTool();
             indexingTool.setConf(new Configuration(hbaseTestUtil.getConfiguration()));
             
-            final String[] cmdArgs = getArgValues(dataTable,indxTable);
+            final String[] cmdArgs = getArgValues(null, dataTable,indxTable);
             int status = indexingTool.run(cmdArgs);
             assertEquals(0, status);
             
             //assert we are pulling from index table.
             rs = conn.createStatement().executeQuery("EXPLAIN " + selectSql);
             actualExplainPlan = QueryUtil.getExplainPlan(rs);
-            assertExplainPlan(actualExplainPlan,dataTable,indxTable,false);
+            assertExplainPlan(actualExplainPlan,null,dataTable,indxTable,false);
             
             rs = stmt.executeQuery(selectSql);
             assertTrue(rs.next());
@@ -242,23 +247,27 @@ public class IndexToolIT {
         }
     }
     
-    private void assertExplainPlan(final String actualExplainPlan, String dataTable,
+    private void assertExplainPlan(final String actualExplainPlan, String schemaName, String
dataTable,
             String indxTable, boolean isLocal) {
         
         String expectedExplainPlan = "";
         if(isLocal) {
-            final String localIndexName = MetaDataUtil.getLocalIndexTableName(dataTable);
+            final String localIndexName = MetaDataUtil.getLocalIndexTableName(SchemaUtil.getTableName(schemaName,
dataTable));
             expectedExplainPlan = String.format("CLIENT 1-CHUNK PARALLEL 1-WAY RANGE SCAN
OVER %s [-32768]"
                 + "\n    SERVER FILTER BY FIRST KEY ONLY", localIndexName);
         } else {
             expectedExplainPlan = String.format("CLIENT 1-CHUNK PARALLEL 1-WAY FULL SCAN
OVER %s"
-                    + "\n    SERVER FILTER BY FIRST KEY ONLY",indxTable);
+                    + "\n    SERVER FILTER BY FIRST KEY ONLY",SchemaUtil.getTableName(schemaName,
indxTable));
         }
         assertEquals(expectedExplainPlan,actualExplainPlan);
     }
 
-    private String[] getArgValues(String dataTable, String indxTable) {
+    private String[] getArgValues(String schemaName, String dataTable, String indxTable)
{
         final List<String> args = Lists.newArrayList();
+        if (schemaName!=null) {
+        	args.add("-s");
+        	args.add(schemaName);
+        }
         args.add("-dt");
         args.add(dataTable);
         args.add("-it");

http://git-wip-us.apache.org/repos/asf/phoenix/blob/3ff71e97/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
index 300f575..d3a1adf 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java
@@ -30,6 +30,7 @@ import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang.exception.ExceptionUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.FileSystem;
@@ -164,12 +165,12 @@ public class IndexTool extends Configured implements Tool {
             final String qIndexTable = SchemaUtil.getTableName(schemaName, indexTable);
          
             connection = ConnectionUtil.getInputConnection(configuration);
-            if(!isValidIndexTable(connection, dataTable, indexTable)) {
+            if(!isValidIndexTable(connection, qDataTable, indexTable)) {
                 throw new IllegalArgumentException(String.format(" %s is not an index table
for %s ",qIndexTable,qDataTable));
             }
             
-            final PTable pdataTable = PhoenixRuntime.getTable(connection, dataTable);
-            final PTable pindexTable = PhoenixRuntime.getTable(connection, indexTable);
+            final PTable pdataTable = PhoenixRuntime.getTable(connection, qDataTable);
+            final PTable pindexTable = PhoenixRuntime.getTable(connection, qIndexTable);
             
             // this is set to ensure index tables remains consistent post population.
             long indxTimestamp = pindexTable.getTimeStamp();
@@ -178,7 +179,7 @@ public class IndexTool extends Configured implements Tool {
             // check if the index type is LOCAL, if so, set the logicalIndexName that is
computed from the dataTable name.
             String logicalIndexTable = qIndexTable;
             if(IndexType.LOCAL.equals(pindexTable.getIndexType())) {
-                logicalIndexTable  = MetaDataUtil.getLocalIndexTableName(dataTable);
+                logicalIndexTable  = MetaDataUtil.getLocalIndexTableName(qDataTable);
             }
             
             final PhoenixConnection pConnection = connection.unwrap(PhoenixConnection.class);
@@ -187,7 +188,7 @@ public class IndexTool extends Configured implements Tool {
             
             final List<String> indexColumns = ddlCompiler.getIndexColumnNames();
             final String selectQuery = ddlCompiler.getSelectQuery();
-            final String upsertQuery = QueryUtil.constructUpsertStatement(indexTable, indexColumns,
Hint.NO_INDEX);
+            final String upsertQuery = QueryUtil.constructUpsertStatement(qIndexTable, indexColumns,
Hint.NO_INDEX);
        
             configuration.set(PhoenixConfigurationUtil.UPSERT_STATEMENT, upsertQuery);
             PhoenixConfigurationUtil.setOutputTableName(configuration, logicalIndexTable);
@@ -231,11 +232,11 @@ public class IndexTool extends Configured implements Tool {
             }
             
             // finally update the index state to ACTIVE.
-            updateIndexState(connection,dataTable,indexTable,PIndexState.ACTIVE);
+            updateIndexState(connection,qDataTable,indexTable,PIndexState.ACTIVE);
             return 0;
             
         } catch (Exception ex) {
-           LOG.error(" An exception occured while performing the indexing job , error message
{} ",ex.getMessage());
+           LOG.error(" An exception occured while performing the indexing job : "+ ExceptionUtils.getStackTrace(ex));
            return -1;
         } finally {
             try {


Mime
View raw message