phoenix-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (PHOENIX-5025) Tool to clean up orphan views
Date Sat, 01 Dec 2018 19:54:00 GMT

    [ https://issues.apache.org/jira/browse/PHOENIX-5025?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16705974#comment-16705974
] 

ASF GitHub Bot commented on PHOENIX-5025:
-----------------------------------------

Github user kadirozde commented on a diff in the pull request:

    https://github.com/apache/phoenix/pull/404#discussion_r238074583
  
    --- Diff: phoenix-core/src/it/java/org/apache/phoenix/end2end/OrphanViewToolIT.java ---
    @@ -0,0 +1,456 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.phoenix.end2end;
    +
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
    +import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
    +import static org.junit.Assert.assertEquals;
    +import static org.junit.Assert.assertTrue;
    +
    +import java.io.IOException;
    +import java.nio.file.Files;
    +import java.nio.file.Paths;
    +import java.sql.Connection;
    +import java.sql.DriverManager;
    +import java.sql.ResultSet;
    +import java.sql.SQLException;
    +import java.util.Arrays;
    +import java.util.Collection;
    +import java.util.List;
    +
    +import com.google.common.collect.Lists;
    +import org.apache.hadoop.conf.Configuration;
    +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
    +import org.apache.phoenix.mapreduce.OrphanViewTool;
    +import org.apache.phoenix.schema.PTable;
    +import org.apache.phoenix.schema.PTableType;
    +import org.apache.phoenix.util.SchemaUtil;
    +import org.junit.Test;
    +import org.junit.runner.RunWith;
    +import org.junit.runners.Parameterized;
    +import org.junit.runners.Parameterized.Parameters;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +@RunWith(Parameterized.class)
    +public class OrphanViewToolIT extends ParallelStatsDisabledIT {
    +    private static final Logger LOG = LoggerFactory.getLogger(OrphanViewToolIT.class);
    +
    +    private final boolean isMultiTenant;
    +    private final boolean columnEncoded;
    +
    +    private static final int fanout = 2;
    +
    +    private static final String filePath = "/tmp/";
    +    private static final String fileName = "/tmp/OrphanView.txt";
    +
    +    protected static String SCHEMA1 = "SCHEMA1";
    +    protected static String SCHEMA2 = "SCHEMA2";
    +    protected static String SCHEMA3 = "SCHEMA3";
    +    protected static String SCHEMA4 = "SCHEMA4";
    +
    +    private final String TENANT_SPECIFIC_URL = getUrl() + ';' + TENANT_ID_ATTRIB + "=tenant";
    +
    +    public OrphanViewToolIT(boolean isMultiTenant, boolean columnEncoded) {
    +        this.isMultiTenant = isMultiTenant;
    +        this.columnEncoded = columnEncoded;
    +    }
    +
    +    @Parameters(name="OrphanViewToolIT_multiTenant={0}, columnEncoded={1}") // name is
used by failsafe as file name in reports
    +    public static Collection<Boolean[]> data() {
    +        return Arrays.asList(new Boolean[][] {
    +                { false, false }, { false, true },
    +                { true, false }, { true, true } });
    +    }
    +
    +    private String generateDDL(String format) {
    +        return generateDDL("", format);
    +    }
    +
    +    private String generateDDL(String options, String format) {
    +        StringBuilder optionsBuilder = new StringBuilder(options);
    +        if (!columnEncoded) {
    +            if (optionsBuilder.length()!=0)
    +                optionsBuilder.append(",");
    +            optionsBuilder.append("COLUMN_ENCODED_BYTES=0");
    +        }
    +        if (isMultiTenant) {
    +            if (optionsBuilder.length()!=0)
    +                optionsBuilder.append(",");
    +            optionsBuilder.append("MULTI_TENANT=true");
    +        }
    +        return String.format(format, isMultiTenant ? "TENANT_ID VARCHAR NOT NULL, " :
"",
    +                isMultiTenant ? "TENANT_ID, " : "", optionsBuilder.toString());
    +    }
    +
    +    private void deleteRows(Connection connection, String systemTableName, String schemaName)
throws SQLException {
    +        String delete = "DELETE FROM " + systemTableName +
    +                " WHERE " + TABLE_SCHEM + " = '" + schemaName + "'";
    +        connection.createStatement().execute(delete);
    +    }
    +
    +    private void deleteAllRows(Connection connection, String baseTableSchema,
    +                            String childViewSchemaName,
    +                            String grandchildViewSchemaName, String grandGrandChildViewSchemaName)
throws SQLException {
    +        deleteRows(connection, SYSTEM_CATALOG_NAME, baseTableSchema);
    +        deleteRows(connection, SYSTEM_CATALOG_NAME, childViewSchemaName);
    +        deleteRows(connection, SYSTEM_CATALOG_NAME, grandchildViewSchemaName);
    +        deleteRows(connection, SYSTEM_CATALOG_NAME, grandGrandChildViewSchemaName);
    +        deleteRows(connection, SYSTEM_CHILD_LINK_NAME, baseTableSchema);
    +        deleteRows(connection, SYSTEM_CHILD_LINK_NAME, childViewSchemaName);
    +        deleteRows(connection, SYSTEM_CHILD_LINK_NAME, grandchildViewSchemaName);
    +        deleteRows(connection, SYSTEM_CHILD_LINK_NAME, grandGrandChildViewSchemaName);
    +
    +        connection.commit();
    +    }
    +    private void createBaseTableAndViews(Connection baseTableConnection, String baseTableFullName,
    +                                         Connection viewConnection, String childViewSchemaName,
    +                                         String grandchildViewSchemaName, String grandGrandChildViewSchemaName)
    +            throws SQLException {
    +        String ddlFormat =
    +                "CREATE TABLE IF NOT EXISTS " + baseTableFullName + "  ("
    +                        + " %s PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR "
    +                        + " CONSTRAINT NAME_PK PRIMARY KEY (%s PK2)" + " ) %s";
    +        baseTableConnection.createStatement().execute(generateDDL(ddlFormat));
    +        // Create a view tree (i.e., tree of views) with depth of 3
    +        for (int i = 0; i < fanout; i++) {
    +            String childView = SchemaUtil.getTableName(childViewSchemaName, generateUniqueName());
    +            String childViewDDL = "CREATE VIEW " + childView + " AS SELECT * FROM " +
baseTableFullName;
    +            viewConnection.createStatement().execute(childViewDDL);
    +            for (int j = 0; j < fanout; j++) {
    +                String grandchildView = SchemaUtil.getTableName(grandchildViewSchemaName,
generateUniqueName());
    +                String grandchildViewDDL = "CREATE VIEW " + grandchildView + " AS SELECT
* FROM " + childView;
    +                viewConnection.createStatement().execute(grandchildViewDDL);
    +                for (int k = 0; k < fanout; k++) {
    +                    viewConnection.createStatement().execute("CREATE VIEW " +
    +                            SchemaUtil.getTableName(grandGrandChildViewSchemaName, generateUniqueName())
+
    +                            " AS SELECT * FROM " + grandchildView);
    +                }
    +            }
    +        }
    +    }
    +
    +    private long getLineCount() throws IOException {
    +        return Files.lines(Paths.get(fileName)).count();
    +    }
    +
    +    @Test
    +    public void testCreateTableAndViews() throws Exception {
    +        String baseTableName = generateUniqueName();
    +        String baseTableFullName = SchemaUtil.getTableName(SCHEMA1, baseTableName);
    +        try (Connection connection = DriverManager.getConnection(getUrl());
    +             Connection viewConnection =
    +                     isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL)
: connection) {
    +            createBaseTableAndViews(connection, baseTableFullName, viewConnection, SCHEMA2,
SCHEMA3, SCHEMA4);
    +            // Run the orphan view tool to drop orphan views but no view should be dropped
    +            runOrphanViewTool(true, false, true, false);
    +            assertTrue(getLineCount() == 0);
    +            // Verify that the views we have created are still in the system catalog
table
    +            String viewQuery = "SELECT COUNT(*) FROM " +
    +                    SYSTEM_CATALOG_NAME +
    +                    " WHERE " + TABLE_TYPE + " = '" + PTableType.VIEW.getSerializedValue()
+ "'";
    +            ResultSet rs = connection.createStatement().executeQuery(viewQuery);
    +            assertTrue(rs.next());
    +            assertTrue(rs.getLong(1) == fanout + fanout * fanout + fanout * fanout *
fanout);
    +            deleteAllRows(connection, SCHEMA1, SCHEMA2, SCHEMA3, SCHEMA4);
    +        }
    +    }
    +
    +    private void verifyNoChildLink(Connection connection, String viewSchemaName) throws
Exception {
    +        // Verify that there there is no link in the system child link table
    +        String childLinkQuery = "SELECT COUNT(*) FROM " + SYSTEM_CHILD_LINK_NAME +
    +                " WHERE " + TABLE_SCHEM + " = '" + viewSchemaName + "' AND " +
    +                LINK_TYPE + " = " + PTable.LinkType.CHILD_TABLE.getSerializedValue();
    +        ResultSet rs = connection.createStatement().executeQuery(childLinkQuery);
    +        assertTrue(rs.next());
    +        assertTrue(rs.getLong(1) == 0);
    +    }
    +
    +    private void verifyNoViewNoLink(Connection connection, String viewSchemaName) throws
Exception {
    +        // Verify that the views and links have been removed from the system catalog
table
    +        String catalogQuery = "SELECT COUNT(*) FROM " + SYSTEM_CATALOG_NAME +
    +                " WHERE " + TABLE_SCHEM + " = '" + viewSchemaName + "'";
    +        ResultSet rs = connection.createStatement().executeQuery(catalogQuery);
    +        assertTrue(rs.next());
    +        long count = rs.getLong(1);
    +        assertTrue(rs.getLong(1) == 0);
    +    }
    +
    +    @Test
    +    public void testDeleteBaseTableRows() throws Exception {
    +        String baseTableName = generateUniqueName();
    +        String baseTableFullName = SchemaUtil.getTableName(SCHEMA1, baseTableName);
    +        try (Connection connection = DriverManager.getConnection(getUrl());
    +             Connection viewConnection =
    +                     isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL)
: connection) {
    +            createBaseTableAndViews(connection, baseTableFullName, viewConnection, SCHEMA2,
SCHEMA2, SCHEMA2);
    +            // Delete the base table rows from the system catalog
    +            String deleteTableRows = "DELETE FROM " + SYSTEM_CATALOG_NAME +
    +                    " WHERE " + TABLE_SCHEM + " = '" + SCHEMA1 + "' AND " +
    +                    TABLE_NAME + " = '" + baseTableName + "' AND " +
    +                    TABLE_TYPE + " = '" + PTableType.TABLE.getSerializedValue() + "'";
    +            connection.createStatement().execute(deleteTableRows);
    +            connection.commit();
    +            // Verify that the views we have created are still in the system catalog
table
    +            String viewQuery = "SELECT COUNT(*) FROM " + SYSTEM_CATALOG_NAME +
    +                    " WHERE " + TABLE_TYPE + " = '" + PTableType.VIEW.getSerializedValue()
+ "'";
    +            ResultSet rs = connection.createStatement().executeQuery(viewQuery);
    +            assertTrue(rs.next());
    +            long count = rs.getLong(1);
    --- End diff --
    
    I will remove it


> Tool to clean up orphan views
> -----------------------------
>
>                 Key: PHOENIX-5025
>                 URL: https://issues.apache.org/jira/browse/PHOENIX-5025
>             Project: Phoenix
>          Issue Type: New Feature
>            Reporter: Kadir OZDEMIR
>            Assignee: Kadir OZDEMIR
>            Priority: Major
>
> A view without its base table is an orphan view. Since views are virtual tables and their
data is stored in their base tables, they are useless when they become orphan. A base table
can have child views, grandchild views and so on. Due to some reasons/bugs, when a base table
was dropped, its views were not not properly cleaned up in the past. For example, the drop
table code did not support cleaning up grandchild views. This has been recently fixed by PHOENIX-4764. Although PHOENIX-4764
prevents new orphan views due to table drop operations, it does not clean up existing orphan
views. It is also believed that when the system catalog table was split due to a bug in the
past, it also contributed to creating orphan views as Phoenix did not support splittable system
catalog. Therefore, Phoenix needs a tool to clean up orphan views.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message