phoenix-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Commented] (PHOENIX-5025) Tool to clean up orphan views
Date Mon, 03 Dec 2018 02:30:00 GMT

    [ https://issues.apache.org/jira/browse/PHOENIX-5025?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16706606#comment-16706606
] 

ASF GitHub Bot commented on PHOENIX-5025:
-----------------------------------------

Github user jpisaac commented on a diff in the pull request:

    https://github.com/apache/phoenix/pull/404#discussion_r238130807
  
    --- Diff: phoenix-core/src/main/java/org/apache/phoenix/mapreduce/OrphanViewTool.java
---
    @@ -0,0 +1,812 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements.  See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership.  The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License.  You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.phoenix.mapreduce;
    +
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_FAMILY;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.LINK_TYPE;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
    +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
    +
    +import java.io.BufferedReader;
    +import java.io.BufferedWriter;
    +import java.io.FileReader;
    +import java.io.FileWriter;
    +import java.io.IOException;
    +import java.sql.Connection;
    +import java.sql.ResultSet;
    +import java.sql.SQLException;
    +import java.util.ArrayList;
    +import java.util.HashMap;
    +import java.util.HashSet;
    +import java.util.Iterator;
    +import java.util.LinkedList;
    +import java.util.List;
    +import java.util.Map;
    +import java.util.Objects;
    +import java.util.Properties;
    +
    +import org.apache.commons.cli.CommandLine;
    +import org.apache.commons.cli.CommandLineParser;
    +import org.apache.commons.cli.HelpFormatter;
    +import org.apache.commons.cli.Option;
    +import org.apache.commons.cli.Options;
    +import org.apache.commons.cli.ParseException;
    +import org.apache.commons.cli.PosixParser;
    +import org.apache.commons.lang.exception.ExceptionUtils;
    +import org.apache.hadoop.conf.Configuration;
    +import org.apache.hadoop.conf.Configured;
    +import org.apache.hadoop.hbase.HBaseConfiguration;
    +import org.apache.hadoop.hbase.TableName;
    +import org.apache.hadoop.util.Tool;
    +import org.apache.hadoop.util.ToolRunner;
    +import org.apache.phoenix.jdbc.PhoenixConnection;
    +import org.apache.phoenix.mapreduce.util.ConnectionUtil;
    +import org.apache.phoenix.parse.DropTableStatement;
    +import org.apache.phoenix.query.QueryServices;
    +import org.apache.phoenix.query.QueryServicesOptions;
    +import org.apache.phoenix.schema.MetaDataClient;
    +import org.apache.phoenix.schema.PTable;
    +import org.apache.phoenix.schema.PTableType;
    +import org.apache.phoenix.schema.TableNotFoundException;
    +import org.apache.phoenix.util.PhoenixRuntime;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;
    +
    +/**
    + * A tool to identify orphan views and links, and drop them
    + *
    + */
    +public class OrphanViewTool extends Configured implements Tool {
    +    private static final Logger LOG = LoggerFactory.getLogger(OrphanViewTool.class);
    +    // Query all the views that are not "MAPPED" views
    +    private static final String viewQuery = "SELECT " +
    +            TENANT_ID + ", " +
    +            TABLE_SCHEM + "," +
    +            TABLE_NAME +
    +            " FROM " + SYSTEM_CATALOG_NAME +
    +            " WHERE "+ TABLE_TYPE + " = '" + PTableType.VIEW.getSerializedValue() +"'
AND NOT " +
    +            VIEW_TYPE + " = " + PTable.ViewType.MAPPED.getSerializedValue();
    +    // Query all physical links
    +    private static final String physicalLinkQuery = "SELECT " +
    +            TENANT_ID + ", " +
    +            TABLE_SCHEM + ", " +
    +            TABLE_NAME + ", " +
    +            COLUMN_NAME + " AS PHYSICAL_TABLE_TENANT_ID, " +
    +            COLUMN_FAMILY + " AS PHYSICAL_TABLE_FULL_NAME " +
    +            " FROM " + SYSTEM_CATALOG_NAME +
    +            " WHERE "+ LINK_TYPE + " = " +
    +            PTable.LinkType.PHYSICAL_TABLE.getSerializedValue();
    +    // Query all child-parent links
    +    private static final String childParentLinkQuery = "SELECT " +
    +            TENANT_ID + ", " +
    +            TABLE_SCHEM + ", " +
    +            TABLE_NAME + ", " +
    +            COLUMN_NAME + " AS PARENT_VIEW_TENANT_ID, " +
    +            COLUMN_FAMILY + " AS PARENT_VIEW_FULL_NAME " +
    +            " FROM " + SYSTEM_CATALOG_NAME +
    +            " WHERE "+ LINK_TYPE + " = " +
    +            PTable.LinkType.PARENT_TABLE.getSerializedValue();
    +    // Query all parent-child links
    +    private static final String parentChildLinkQuery = "SELECT " +
    +            TENANT_ID + ", " +
    +            TABLE_SCHEM + ", " +
    +            TABLE_NAME + ", " +
    +            COLUMN_NAME + " AS CHILD_VIEW_TENANT_ID, " +
    +            COLUMN_FAMILY + " AS CHILD_VIEW_FULL_NAME " +
    +            " FROM " + SYSTEM_CHILD_LINK_NAME +
    +            " WHERE "+ LINK_TYPE + " = " +
    +            PTable.LinkType.CHILD_TABLE.getSerializedValue();
    +
    +    // Query all the tables that can be a base table
    +    private static final String candidateBaseTableQuery = "SELECT " +
    +            TENANT_ID + ", " +
    +            TABLE_SCHEM + ", " +
    +            TABLE_NAME +
    +            " FROM " + SYSTEM_CATALOG_NAME +
    +            " WHERE "+ " NOT " + TABLE_TYPE + " = '" + PTableType.VIEW.getSerializedValue()
+ "'";
    +
    +    private String outputPath;
    +    private String inputPath;
    +    private boolean clean = false;
    +    private int maxViewLevel = 0;
    +    private static final long defaultAge = 24*60*60*1000; // 1 day
    +    private long age = 0;
    +
    +    private static final byte VIEW = 0;
    +    private static final byte PHYSICAL_TABLE_LINK = 1;
    +    private static final byte PARENT_TABLE_LINK = 2;
    +    private static final byte CHILD_TABLE_LINK = 3;
    +    private static final byte ORPHAN_TYPE_COUNT = 4;
    +
    +    BufferedWriter writer[] = new BufferedWriter[ORPHAN_TYPE_COUNT];
    +    BufferedReader reader[] = new BufferedReader[ORPHAN_TYPE_COUNT];
    +    HashMap<Key, View> orphanViewSet = new HashMap<>();
    +    List<HashMap<Key, View>> viewSetArray = new ArrayList<HashMap<Key,
View>>();
    +    HashMap<Key, Base> baseSet = new HashMap<>();
    +    HashSet<Link> orphanLinkSet = new HashSet<>();
    +
    +    private static final String fileName[] = {"OrphanView.txt", "OrphanPhysicalTableLink.txt",
"OrphanParentTableLink", "OrphanChildTableLink"};
    +    private static final Option OUTPUT_PATH_OPTION = new Option("op", "output-path",
true,
    +            "Output path where the files listing orphan views and links are written");
    +    private static final Option INPUT_PATH_OPTION = new Option("ip", "input-path", true,
    +            "Input path where the files listing orphan views and links are read");
    +    private static final Option CLEAN_ORPHAN_VIEWS_OPTION = new Option("c", "clean",
false,
    +            "If specified, cleans orphan views and links");
    +    private static final Option IDENTIFY_ORPHAN_VIEWS_OPTION = new Option("i", "identify",
false,
    +            "If specified, identifies orphan views and links");
    +    private static final Option AGE_OPTION = new Option("a", "age", true,
    +            "The minimum age (in milliseconds) for the views (default value is " + Long.toString(defaultAge)
+ ", i.e. 1 day)");
    +    private static final Option HELP_OPTION = new Option("h", "help", false, "Help");
    +
    +    private Options getOptions() {
    +        final Options options = new Options();
    +        options.addOption(OUTPUT_PATH_OPTION);
    +        options.addOption(INPUT_PATH_OPTION);
    +        options.addOption(CLEAN_ORPHAN_VIEWS_OPTION);
    +        options.addOption(IDENTIFY_ORPHAN_VIEWS_OPTION);
    +        options.addOption(AGE_OPTION);
    +        options.addOption(HELP_OPTION);
    +        return options;
    +    }
    +
    +    /**
    +     * Parses the commandline arguments, throws IllegalStateException if mandatory arguments
are
    +     * missing.
    +     * @param args supplied command line arguments
    +     */
    +    private void parseOptions(String[] args) throws Exception {
    +
    +        final Options options = getOptions();
    +
    +        CommandLineParser parser = new PosixParser();
    +        CommandLine cmdLine = null;
    +        try {
    +            cmdLine = parser.parse(options, args);
    +        } catch (ParseException e) {
    +            printHelpAndExit("Error parsing command line options: " + e.getMessage(),
options);
    +        }
    +        if (cmdLine.hasOption(HELP_OPTION.getOpt())) {
    +            printHelpAndExit(options, 0);
    +        }
    +        if (cmdLine.hasOption(OUTPUT_PATH_OPTION.getOpt()) && cmdLine.hasOption(INPUT_PATH_OPTION.getOpt()))
{
    +            throw new IllegalStateException("Specify either " + OUTPUT_PATH_OPTION.getLongOpt()
+ " or "
    +                    + INPUT_PATH_OPTION.getOpt());
    +        }
    +        if (cmdLine.hasOption(INPUT_PATH_OPTION.getOpt()) && !cmdLine.hasOption(CLEAN_ORPHAN_VIEWS_OPTION.getOpt()))
{
    +            throw new IllegalStateException(INPUT_PATH_OPTION.getLongOpt() + " is only
used with "
    +                    + IDENTIFY_ORPHAN_VIEWS_OPTION.getOpt());
    +        }
    +        if (cmdLine.hasOption(IDENTIFY_ORPHAN_VIEWS_OPTION.getOpt()) && cmdLine.hasOption(CLEAN_ORPHAN_VIEWS_OPTION.getOpt()))
{
    +            throw new IllegalStateException("Specify either " + IDENTIFY_ORPHAN_VIEWS_OPTION.getLongOpt()
+ " or "
    +                    + IDENTIFY_ORPHAN_VIEWS_OPTION.getOpt());
    +        }
    +        if (cmdLine.hasOption(OUTPUT_PATH_OPTION.getOpt()) && (!cmdLine.hasOption(IDENTIFY_ORPHAN_VIEWS_OPTION.getOpt())
&&
    +                !cmdLine.hasOption(CLEAN_ORPHAN_VIEWS_OPTION.getOpt()))) {
    +            throw new IllegalStateException(OUTPUT_PATH_OPTION.getLongOpt() + " requires
either " +
    +                    IDENTIFY_ORPHAN_VIEWS_OPTION.getOpt() + " or " + CLEAN_ORPHAN_VIEWS_OPTION.getOpt());
    +        }
    +        if (cmdLine.hasOption(CLEAN_ORPHAN_VIEWS_OPTION.getOpt())) {
    +            clean = true;
    +        }
    +        else if (!cmdLine.hasOption(IDENTIFY_ORPHAN_VIEWS_OPTION.getOpt())) {
    +            throw new IllegalStateException("Specify either " +
    +                    IDENTIFY_ORPHAN_VIEWS_OPTION.getOpt() + " or " + CLEAN_ORPHAN_VIEWS_OPTION.getOpt());
    +        }
    +        if (cmdLine.hasOption(AGE_OPTION.getOpt())) {
    +            age = Long.valueOf(cmdLine.getOptionValue(AGE_OPTION.getOpt()));
    +        }
    +
    +        outputPath = cmdLine.getOptionValue(OUTPUT_PATH_OPTION.getOpt());
    +        inputPath = cmdLine.getOptionValue(INPUT_PATH_OPTION.getOpt());
    +    }
    +
    +    private void printHelpAndExit(String errorMessage, Options options) {
    +        System.err.println(errorMessage);
    +        printHelpAndExit(options, 1);
    +    }
    +
    +    private void printHelpAndExit(Options options, int exitCode) {
    +        HelpFormatter formatter = new HelpFormatter();
    +        formatter.printHelp("help", options);
    +        System.exit(exitCode);
    +    }
    +
    +    private class Key {
    +        private String tenantId;
    +        private String schemaName;
    +        private String tableName;
    +
    +        public Key (String tenantId, String schemaName, String tableName) {
    +            this.tenantId = tenantId;
    --- End diff --
    
    Check for Preconditions and reduce the member variables to its canonical form as expected
in the rest of the code if the preconditions are not met. 
    Applies to other constructors of Key.
    



> Tool to clean up orphan views
> -----------------------------
>
>                 Key: PHOENIX-5025
>                 URL: https://issues.apache.org/jira/browse/PHOENIX-5025
>             Project: Phoenix
>          Issue Type: New Feature
>            Reporter: Kadir OZDEMIR
>            Assignee: Kadir OZDEMIR
>            Priority: Major
>
> A view without its base table is an orphan view. Since views are virtual tables and their
data is stored in their base tables, they are useless when they become orphan. A base table
can have child views, grandchild views and so on. Due to some reasons/bugs, when a base table
was dropped, its views were not not properly cleaned up in the past. For example, the drop
table code did not support cleaning up grandchild views. This has been recently fixed by PHOENIX-4764. Although PHOENIX-4764
prevents new orphan views due to table drop operations, it does not clean up existing orphan
views. It is also believed that when the system catalog table was split due to a bug in the
past, it also contributed to creating orphan views as Phoenix did not support splittable system
catalog. Therefore, Phoenix needs a tool to clean up orphan views.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message