phoenix-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kadirozde <...@git.apache.org>
Subject [GitHub] phoenix pull request #404: PHOENIX-5025 Tool to clean up orphan views
Date Sat, 01 Dec 2018 19:53:42 GMT
Github user kadirozde commented on a diff in the pull request:

    https://github.com/apache/phoenix/pull/404#discussion_r238074583
  
    --- Diff: phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java ---
    @@ -0,0 +1,456 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.phoenix.end2end;
    +
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
    +import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
    +import static org.junit.Assert.assertEquals;
    +import static org.junit.Assert.assertTrue;
    +
    +import java.io.IOException;
    +import java.nio.file.Files;
    +import java.nio.file.Paths;
    +import java.sql.Connection;
    +import java.sql.DriverManager;
    +import java.sql.ResultSet;
    +import java.sql.SQLException;
    +import java.util.Arrays;
    +import java.util.Collection;
    +import java.util.List;
    +
    +import com.google.common.collect.Lists;
    +import org.apache.hadoop.conf.Configuration;
    +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
    +import org.apache.phoenix.mapreduce.OrphanViewTool;
    +import org.apache.phoenix.schema.PTable;
    +import org.apache.phoenix.schema.PTableType;
    +import org.apache.phoenix.util.SchemaUtil;
    +import org.junit.Test;
    +import org.junit.runner.RunWith;
    +import org.junit.runners.Parameterized;
    +import org.junit.runners.Parameterized.Parameters;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +@RunWith(Parameterized.class)
    +public class OrphanViewToolIT extends ParallelStatsDisabledIT {
    +    private static final Logger LOG = LoggerFactory.getLogger(OrphanViewToolIT.class);
    +
    +    private final boolean isMultiTenant;
    +    private final boolean columnEncoded;
    +
    +    private static final int fanout = 2;
    +
    +    private static final String filePath = "/tmp/";
    +    private static final String fileName = "/tmp/OrphanView.txt";
    +
    +    protected static String SCHEMA1 = "SCHEMA1";
    +    protected static String SCHEMA2 = "SCHEMA2";
    +    protected static String SCHEMA3 = "SCHEMA3";
    +    protected static String SCHEMA4 = "SCHEMA4";
    +
    +    private final String TENANT_SPECIFIC_URL = getUrl() + ';' + TENANT_ID_ATTRIB + "=tenant";
    +
    +    public OrphanViewToolIT(boolean isMultiTenant, boolean columnEncoded) {
    +        this.isMultiTenant = isMultiTenant;
    +        this.columnEncoded = columnEncoded;
    +    }
    +
    +    @Parameters(name="OrphanViewToolIT_multiTenant={0}, columnEncoded={1}") // name is
used by failsafe as file name in reports
    +    public static Collection<Boolean[]> data() {
    +        return Arrays.asList(new Boolean[][] {
    +                { false, false }, { false, true },
    +                { true, false }, { true, true } });
    +    }
    +
    +    private String generateDDL(String format) {
    +        return generateDDL("", format);
    +    }
    +
    +    private String generateDDL(String options, String format) {
    +        StringBuilder optionsBuilder = new StringBuilder(options);
    +        if (!columnEncoded) {
    +            if (optionsBuilder.length()!=0)
    +                optionsBuilder.append(",");
    +            optionsBuilder.append("COLUMN_ENCODED_BYTES=0");
    +        }
    +        if (isMultiTenant) {
    +            if (optionsBuilder.length()!=0)
    +                optionsBuilder.append(",");
    +            optionsBuilder.append("MULTI_TENANT=true");
    +        }
    +        return String.format(format, isMultiTenant ? "TENANT_ID VARCHAR NOT NULL, " :
"",
    +                isMultiTenant ? "TENANT_ID, " : "", optionsBuilder.toString());
    +    }
    +
    +    private void deleteRows(Connection connection, String systemTableName, String schemaName)
throws SQLException {
    +        String delete = "DELETE FROM " + systemTableName +
    +                " WHERE " + TABLE_SCHEM + " = '" + schemaName + "'";
    +        connection.createStatement().execute(delete);
    +    }
    +
    +    private void deleteAllRows(Connection connection, String baseTableSchema,
    +                            String childViewSchemaName,
    +                            String grandchildViewSchemaName, String grandGrandChildViewSchemaName)
throws SQLException {
    +        deleteRows(connection, SYSTEM_CATALOG_NAME, baseTableSchema);
    +        deleteRows(connection, SYSTEM_CATALOG_NAME, childViewSchemaName);
    +        deleteRows(connection, SYSTEM_CATALOG_NAME, grandchildViewSchemaName);
    +        deleteRows(connection, SYSTEM_CATALOG_NAME, grandGrandChildViewSchemaName);
    +        deleteRows(connection, SYSTEM_CHILD_LINK_NAME, baseTableSchema);
    +        deleteRows(connection, SYSTEM_CHILD_LINK_NAME, childViewSchemaName);
    +        deleteRows(connection, SYSTEM_CHILD_LINK_NAME, grandchildViewSchemaName);
    +        deleteRows(connection, SYSTEM_CHILD_LINK_NAME, grandGrandChildViewSchemaName);
    +
    +        connection.commit();
    +    }
    +    private void createBaseTableAndViews(Connection baseTableConnection, String baseTableFullName,
    +                                         Connection viewConnection, String childViewSchemaName,
    +                                         String grandchildViewSchemaName, String grandGrandChildViewSchemaName)
    +            throws SQLException {
    +        String ddlFormat =
    +                "CREATE TABLE IF NOT EXISTS " + baseTableFullName + "  ("
    +                        + " %s PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR "
    +                        + " CONSTRAINT NAME_PK PRIMARY KEY (%s PK2)" + " ) %s";
    +        baseTableConnection.createStatement().execute(generateDDL(ddlFormat));
    +        // Create a view tree (i.e., tree of views) with depth of 3
    +        for (int i = 0; i < fanout; i++) {
    +            String childView = SchemaUtil.getTableName(childViewSchemaName, generateUniqueName());
    +            String childViewDDL = "CREATE VIEW " + childView + " AS SELECT * FROM " +
baseTableFullName;
    +            viewConnection.createStatement().execute(childViewDDL);
    +            for (int j = 0; j < fanout; j++) {
    +                String grandchildView = SchemaUtil.getTableName(grandchildViewSchemaName,
generateUniqueName());
    +                String grandchildViewDDL = "CREATE VIEW " + grandchildView + " AS SELECT
* FROM " + childView;
    +                viewConnection.createStatement().execute(grandchildViewDDL);
    +                for (int k = 0; k < fanout; k++) {
    +                    viewConnection.createStatement().execute("CREATE VIEW " +
    +                            SchemaUtil.getTableName(grandGrandChildViewSchemaName, generateUniqueName())
+
    +                            " AS SELECT * FROM " + grandchildView);
    +                }
    +            }
    +        }
    +    }
    +
    +    private long getLineCount() throws IOException {
    +        return Files.lines(Paths.get(fileName)).count();
    +    }
    +
    +    @Test
    +    public void testCreateTableAndViews() throws Exception {
    +        String baseTableName = generateUniqueName();
    +        String baseTableFullName = SchemaUtil.getTableName(SCHEMA1, baseTableName);
    +        try (Connection connection = DriverManager.getConnection(getUrl());
    +             Connection viewConnection =
    +                     isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL)
: connection) {
    +            createBaseTableAndViews(connection, baseTableFullName, viewConnection, SCHEMA2,
SCHEMA3, SCHEMA4);
    +            // Run the orphan view tool to drop orphan views but no view should be dropped
    +            runOrphanViewTool(true, false, true, false);
    +            assertTrue(getLineCount() == 0);
    +            // Verify that the views we have created are still in the system catalog
table
    +            String viewQuery = "SELECT COUNT(*) FROM " +
    +                    SYSTEM_CATALOG_NAME +
    +                    " WHERE " + TABLE_TYPE + " = '" + PTableType.VIEW.getSerializedValue()
+ "'";
    +            ResultSet rs = connection.createStatement().executeQuery(viewQuery);
    +            assertTrue(rs.next());
    +            assertTrue(rs.getLong(1) == fanout + fanout * fanout + fanout * fanout *
fanout);
    +            deleteAllRows(connection, SCHEMA1, SCHEMA2, SCHEMA3, SCHEMA4);
    +        }
    +    }
    +
    +    private void verifyNoChildLink(Connection connection, String viewSchemaName) throws
Exception {
    +        // Verify that there there is no link in the system child link table
    +        String childLinkQuery = "SELECT COUNT(*) FROM " + SYSTEM_CHILD_LINK_NAME +
    +                " WHERE " + TABLE_SCHEM + " = '" + viewSchemaName + "' AND " +
    +                LINK_TYPE + " = " + PTable.LinkType.CHILD_TABLE.getSerializedValue();
    +        ResultSet rs = connection.createStatement().executeQuery(childLinkQuery);
    +        assertTrue(rs.next());
    +        assertTrue(rs.getLong(1) == 0);
    +    }
    +
    +    private void verifyNoViewNoLink(Connection connection, String viewSchemaName) throws
Exception {
    +        // Verify that the views and links have been removed from the system catalog
table
    +        String catalogQuery = "SELECT COUNT(*) FROM " + SYSTEM_CATALOG_NAME +
    +                " WHERE " + TABLE_SCHEM + " = '" + viewSchemaName + "'";
    +        ResultSet rs = connection.createStatement().executeQuery(catalogQuery);
    +        assertTrue(rs.next());
    +        long count = rs.getLong(1);
    +        assertTrue(rs.getLong(1) == 0);
    +    }
    +
    +    @Test
    +    public void testDeleteBaseTableRows() throws Exception {
    +        String baseTableName = generateUniqueName();
    +        String baseTableFullName = SchemaUtil.getTableName(SCHEMA1, baseTableName);
    +        try (Connection connection = DriverManager.getConnection(getUrl());
    +             Connection viewConnection =
    +                     isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL)
: connection) {
    +            createBaseTableAndViews(connection, baseTableFullName, viewConnection, SCHEMA2,
SCHEMA2, SCHEMA2);
    +            // Delete the base table rows from the system catalog
    +            String deleteTableRows = "DELETE FROM " + SYSTEM_CATALOG_NAME +
    +                    " WHERE " + TABLE_SCHEM + " = '" + SCHEMA1 + "' AND " +
    +                    TABLE_NAME + " = '" + baseTableName + "' AND " +
    +                    TABLE_TYPE + " = '" + PTableType.TABLE.getSerializedValue() + "'";
    +            connection.createStatement().execute(deleteTableRows);
    +            connection.commit();
    +            // Verify that the views we have created are still in the system catalog
table
    +            String viewQuery = "SELECT COUNT(*) FROM " + SYSTEM_CATALOG_NAME +
    +                    " WHERE " + TABLE_TYPE + " = '" + PTableType.VIEW.getSerializedValue()
+ "'";
    +            ResultSet rs = connection.createStatement().executeQuery(viewQuery);
    +            assertTrue(rs.next());
    +            long count = rs.getLong(1);
    --- End diff --
    
    I will remove it


---

Mime
View raw message