phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jamestay...@apache.org
Subject [6/6] phoenix git commit: PHOENIX-4700 Fix split policy on system tables other than SYSTEM.CATALOG
Date Thu, 26 Apr 2018 17:35:29 GMT
PHOENIX-4700 Fix split policy on system tables other than SYSTEM.CATALOG


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/338611cd
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/338611cd
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/338611cd

Branch: refs/heads/4.x-HBase-0.98
Commit: 338611cdbddc75988e1d95b244a322f10900b742
Parents: 532447a
Author: James Taylor <jtaylor@salesforce.com>
Authored: Mon Apr 23 10:14:36 2018 -0700
Committer: James Taylor <jtaylor@salesforce.com>
Committed: Thu Apr 26 10:35:06 2018 -0700

----------------------------------------------------------------------
 .../apache/phoenix/query/QueryConstants.java    |  8 +-
 .../SplitOnLeadingVarCharColumnsPolicy.java     | 43 +++++++++
 .../schema/SystemFunctionSplitPolicy.java       | 27 ++++++
 .../phoenix/schema/SystemStatsSplitPolicy.java  | 27 ++++++
 .../phoenix/schema/SystemSplitPolicyTest.java   | 97 ++++++++++++++++++++
 5 files changed, 198 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/338611cd/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index 1c7e06c..d7fd8e6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -34,6 +34,8 @@ import org.apache.phoenix.schema.PName;
 import org.apache.phoenix.schema.PNameFactory;
 import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
 import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.SystemFunctionSplitPolicy;
+import org.apache.phoenix.schema.SystemStatsSplitPolicy;
 import org.apache.phoenix.schema.TableProperty;
 
 
@@ -257,7 +259,7 @@ public interface QueryConstants {
             + PHYSICAL_NAME + ","
             + COLUMN_FAMILY + ","+ GUIDE_POST_KEY+"))\n" +
             // Install split policy to prevent a physical table's stats from being split
across regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() +
"',\n" + 
+            HTableDescriptor.SPLIT_POLICY + "='" + SystemStatsSplitPolicy.class.getName()
+ "',\n" + 
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
 
     public static final String CREATE_SEQUENCE_METADATA =
@@ -301,7 +303,7 @@ public interface QueryConstants {
             HConstants.VERSIONS + "=%s,\n" +
             HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n"+
             // Install split policy to prevent a tenant's metadata from being split across
regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() +
"',\n" + 
+            HTableDescriptor.SPLIT_POLICY + "='" + SystemFunctionSplitPolicy.class.getName()
+ "',\n" + 
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
     public static final String CREATE_LOG_METADATA =
             "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_LOG_TABLE + "\"(\n"
+
@@ -325,8 +327,6 @@ public interface QueryConstants {
             " CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (QUERY_ID))\n" +
             HConstants.VERSIONS + "= " + MetaDataProtocol.DEFAULT_LOG_VERSIONS + ",\n" +
             HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n"+
-            // Install split policy to prevent a tenant's metadata from being split across
regions.
-            HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() +
"',\n" +
             PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE+ ",\n" +
             HColumnDescriptor.TTL + "=" + MetaDataProtocol.DEFAULT_LOG_TTL+",\n"+
             TableProperty.COLUMN_ENCODED_BYTES.toString()+" = 0";

http://git-wip-us.apache.org/repos/asf/phoenix/blob/338611cd/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java
new file mode 100644
index 0000000..d481998
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
+import org.apache.phoenix.util.SchemaUtil;
+
+public abstract class SplitOnLeadingVarCharColumnsPolicy extends ConstantSizeRegionSplitPolicy
{
+    abstract protected int getColumnToSplitAt();
+    
+    protected final byte[] getSplitPoint(byte[] splitPoint) {
+        int offset = SchemaUtil.getVarCharLength(splitPoint, 0, splitPoint.length, getColumnToSplitAt());
+        // Only split between leading columns indicated.
+        if (offset == splitPoint.length) {
+            return splitPoint;
+        }
+        // Otherwise, an attempt is being made to split in the middle of a table.
+        // Just return a split point at the boundary of the first two columns instead
+        byte[] newSplitPoint = new byte[offset + 1];
+        System.arraycopy(splitPoint, 0, newSplitPoint, 0, offset+1);
+        return newSplitPoint;
+    }
+    
+    @Override
+    protected final byte[] getSplitPoint() {
+        return getSplitPoint(super.getSplitPoint());
+    }
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/338611cd/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemFunctionSplitPolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemFunctionSplitPolicy.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemFunctionSplitPolicy.java
new file mode 100644
index 0000000..58e1f9f
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemFunctionSplitPolicy.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+public class SystemFunctionSplitPolicy extends SplitOnLeadingVarCharColumnsPolicy {
+
+    @Override
+    protected int getColumnToSplitAt() {
+        return 2;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/338611cd/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemStatsSplitPolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemStatsSplitPolicy.java
b/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemStatsSplitPolicy.java
new file mode 100644
index 0000000..69fe8aa
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/SystemStatsSplitPolicy.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+public class SystemStatsSplitPolicy extends SplitOnLeadingVarCharColumnsPolicy {
+
+    @Override
+    protected int getColumnToSplitAt() {
+        return 1;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/phoenix/blob/338611cd/phoenix-core/src/test/java/org/apache/phoenix/schema/SystemSplitPolicyTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/SystemSplitPolicyTest.java
b/phoenix-core/src/test/java/org/apache/phoenix/schema/SystemSplitPolicyTest.java
new file mode 100644
index 0000000..01074b4
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/SystemSplitPolicyTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+import static org.junit.Assert.assertArrayEquals;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.stats.StatisticsUtil;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.schema.types.PVarchar;
+import org.apache.phoenix.util.ByteUtil;
+import org.junit.Test;
+
+public class SystemSplitPolicyTest {
+    @Test
+    public void testStatsSplitPolicy() {
+        SplitOnLeadingVarCharColumnsPolicy policy = new SystemStatsSplitPolicy();
+        byte[] splitOn;
+        byte[] rowKey;
+        byte[] table;
+        ImmutableBytesWritable family;
+        table = PVarchar.INSTANCE.toBytes("FOO.BAR");
+        family = QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES_PTR;
+        rowKey = ByteUtil.concat(
+                PLong.INSTANCE.toBytes(20L), 
+                PVarchar.INSTANCE.toBytes("BAS"), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY, 
+                PInteger.INSTANCE.toBytes(100));
+        splitOn = StatisticsUtil.getRowKey(table, family, rowKey);
+        splitOn = policy.getSplitPoint(splitOn);
+        assertArrayEquals(ByteUtil.concat(table, QueryConstants.SEPARATOR_BYTE_ARRAY), splitOn);
+        
+        table = PVarchar.INSTANCE.toBytes("MY_TABLE");
+        family = new ImmutableBytesWritable(Bytes.toBytes("ABC"));
+        rowKey = ByteUtil.concat(
+                PVarchar.INSTANCE.toBytes("BAS"), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY, 
+                PInteger.INSTANCE.toBytes(100),
+                PLong.INSTANCE.toBytes(20L));
+        splitOn = StatisticsUtil.getRowKey(table, family, rowKey);
+        splitOn = policy.getSplitPoint(splitOn);
+        assertArrayEquals(ByteUtil.concat(table, QueryConstants.SEPARATOR_BYTE_ARRAY), splitOn);
+    }
+    
+    private static byte[] getSystemFunctionRowKey(String tenantId, String funcName, String
typeName, byte[] argPos) {
+        return ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY,
+                PVarchar.INSTANCE.toBytes(funcName),
+                QueryConstants.SEPARATOR_BYTE_ARRAY,
+                PVarchar.INSTANCE.toBytes(typeName), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY,
+                argPos
+                );
+    }
+    
+    private static byte[] getSystemFunctionSplitKey(String tenantId, String funcName) {
+        return ByteUtil.concat(PVarchar.INSTANCE.toBytes(tenantId), 
+                QueryConstants.SEPARATOR_BYTE_ARRAY,
+                PVarchar.INSTANCE.toBytes(funcName),
+                QueryConstants.SEPARATOR_BYTE_ARRAY);
+    }
+    
+    @Test
+    public void testFunctionSplitPolicy() {
+        SplitOnLeadingVarCharColumnsPolicy policy = new SystemFunctionSplitPolicy();
+        byte[] splitPoint;
+        byte[] rowKey;
+        byte[] expectedSplitPoint;
+        rowKey = getSystemFunctionRowKey("","MY_FUNC", "VARCHAR", Bytes.toBytes(3));
+        expectedSplitPoint = getSystemFunctionSplitKey("","MY_FUNC");
+        splitPoint = policy.getSplitPoint(rowKey);
+        assertArrayEquals(expectedSplitPoint, splitPoint);
+        
+        rowKey = getSystemFunctionRowKey("TENANT1","F", "", Bytes.toBytes(3));
+        expectedSplitPoint = getSystemFunctionSplitKey("TENANT1","F");
+        splitPoint = policy.getSplitPoint(rowKey);
+        assertArrayEquals(expectedSplitPoint, splitPoint);
+    }
+}


Mime
View raw message