From commits-return-31174-archive-asf-public=cust-asf.ponee.io@hive.apache.org Fri Jan 12 05:44:15 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id 17688180656 for ; Fri, 12 Jan 2018 05:44:15 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 07A7D160C42; Fri, 12 Jan 2018 04:44:15 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 03F5B160C13 for ; Fri, 12 Jan 2018 05:44:12 +0100 (CET) Received: (qmail 39629 invoked by uid 500); 12 Jan 2018 04:44:12 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 39618 invoked by uid 99); 12 Jan 2018 04:44:11 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 12 Jan 2018 04:44:11 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AB8A8E9457; Fri, 12 Jan 2018 04:44:11 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: anishek@apache.org To: commits@hive.apache.org Date: Fri, 12 Jan 2018 04:44:11 -0000 Message-Id: <6891ba0d46244aa5991cdce602f18aaa@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] hive git commit: HIVE-18352: introduce a METADATAONLY option while doing REPL DUMP to allow integrations of other tools (Anishek Agarwal reviewed by Daniel Dai) Repository: hive Updated Branches: refs/heads/master 5f25083ba -> fd4e222d5 http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/exec/util/DAGTraversal.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/util/DAGTraversal.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/util/DAGTraversal.java index 1e436ba..6dce835 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/util/DAGTraversal.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/util/DAGTraversal.java @@ -1,19 +1,19 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.ql.exec.util; http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java index 4ab7312..9a1dc97 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; import org.apache.hadoop.hive.ql.parse.repl.dump.io.DBSerializer; @@ -261,16 +262,15 @@ public class EximUtil { } } - public static void createExportDump(FileSystem fs, Path metadataPath, - org.apache.hadoop.hive.ql.metadata.Table tableHandle, - Iterable partitions, - ReplicationSpec replicationSpec) throws SemanticException, IOException { + public static void createExportDump(FileSystem fs, Path metadataPath, Table tableHandle, + Iterable partitions, ReplicationSpec replicationSpec, HiveConf hiveConf) + throws SemanticException, IOException { - if (replicationSpec == null){ + if (replicationSpec == null) { replicationSpec = new ReplicationSpec(); // instantiate default values if not specified } - if (tableHandle == null){ + if (tableHandle == null) { replicationSpec.setNoop(true); } @@ -278,7 +278,7 @@ public class EximUtil { if (replicationSpec.isInReplicationScope()) { new ReplicationSpecSerializer().writeTo(writer, replicationSpec); } - new TableSerializer(tableHandle, partitions).writeTo(writer, replicationSpec); + new TableSerializer(tableHandle, partitions, hiveConf).writeTo(writer, replicationSpec); } } @@ -404,50 +404,4 @@ public class EximUtil { } }; } - - /** - * Verify if a table should be exported or not - */ - public static Boolean shouldExportTable(ReplicationSpec replicationSpec, Table tableHandle) throws SemanticException { - if (replicationSpec == null) - { - replicationSpec = new ReplicationSpec(); - } - - if (replicationSpec.isNoop()) - { - return false; - } - - if (tableHandle == null) - { - return false; - } - - if (replicationSpec.isInReplicationScope()) { - return !(tableHandle == null || tableHandle.isTemporary() || tableHandle.isNonNative() || - (tableHandle.getParameters() != null && StringUtils.equals(tableHandle.getParameters().get("transactional"), "true"))); - } - - if (tableHandle.isNonNative()) { - throw new SemanticException(ErrorMsg.EXIM_FOR_NON_NATIVE.getMsg()); - } - - return true; - } - - /** - * Verify if a table should be exported or not by talking to metastore to fetch table info. - * Return true when running into errors with metastore call. - */ - public static Boolean tryValidateShouldExportTable(Hive db, String dbName, String tableName, ReplicationSpec replicationSpec) { - try { - Table table = db.getTable(dbName, tableName); - return EximUtil.shouldExportTable(replicationSpec, table); - } catch (Exception e) { - // Swallow the exception - LOG.error("Failed to validate if the table should be exported or not", e); - } - return true; - } } http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java index b253236..ef3e80d2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExportSemanticAnalyzer.java @@ -1,19 +1,19 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.ql.parse; http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g index 209f75d..3360c0e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g @@ -879,7 +879,8 @@ replDumpStatement (KW_TO (rangeEnd=Number))? (KW_LIMIT (batchSize=Number))? )? - -> ^(TOK_REPL_DUMP $dbName $tblName? ^(TOK_FROM $eventId (TOK_TO $rangeEnd)? (TOK_LIMIT $batchSize)?)? ) + (KW_WITH replConf=replConfigs)? + -> ^(TOK_REPL_DUMP $dbName ^(TOK_TABNAME $tblName)? ^(TOK_FROM $eventId (TOK_TO $rangeEnd)? (TOK_LIMIT $batchSize)?)? $replConf?) ; replLoadStatement http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java index 7a28b43..197f219 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IHMSHandler; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.slf4j.Logger; @@ -86,7 +87,8 @@ public class MetaDataExportListener extends MetaStorePreEventListener { Path outFile = new Path(metaPath, name + EximUtil.METADATA_NAME); try { SessionState.getConsole().printInfo("Beginning metadata export"); - EximUtil.createExportDump(fs, outFile, mTbl, null, null); + EximUtil.createExportDump(fs, outFile, mTbl, null, null, + new HiveConf(conf, MetaDataExportListener.class)); if (moveMetadataToTrash == true) { wh.deleteDir(metaPath, true); } http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java index 80556ae..1d89661 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java @@ -1,19 +1,19 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.ql.parse; @@ -34,13 +34,13 @@ import org.apache.hadoop.hive.ql.exec.repl.bootstrap.ReplLoadWork; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.repl.ReplLogger; import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; import org.apache.hadoop.hive.ql.parse.repl.load.EventDumpDirComparator; import org.apache.hadoop.hive.ql.parse.repl.load.UpdatedMetaDataTracker; -import org.apache.hadoop.hive.ql.parse.repl.load.message.MessageHandler; import org.apache.hadoop.hive.ql.parse.repl.load.log.IncrementalLoadLogger; -import org.apache.hadoop.hive.ql.parse.repl.ReplLogger; +import org.apache.hadoop.hive.ql.parse.repl.load.message.MessageHandler; import org.apache.hadoop.hive.ql.plan.AlterDatabaseDesc; import org.apache.hadoop.hive.ql.plan.AlterTableDesc; import org.apache.hadoop.hive.ql.plan.DDLWork; @@ -59,7 +59,6 @@ import java.util.List; import java.util.Map; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_DBNAME; -import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_FROM; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_LIMIT; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_CONFIG; import static org.apache.hadoop.hive.ql.parse.HiveParser.TOK_REPL_DUMP; @@ -128,9 +127,17 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { // skip the first node, which is always required int currNode = 1; while (currNode < numChildren) { - if (ast.getChild(currNode).getType() != TOK_FROM) { + if (ast.getChild(currNode).getType() == TOK_REPL_CONFIG) { + Map replConfigs + = DDLSemanticAnalyzer.getProps((ASTNode) ast.getChild(currNode).getChild(0)); + if (null != replConfigs) { + for (Map.Entry config : replConfigs.entrySet()) { + conf.set(config.getKey(), config.getValue()); + } + } + } else if (ast.getChild(currNode).getType() == TOK_TABNAME) { // optional tblName was specified. - tblNameOrPattern = PlanUtils.stripQuotes(ast.getChild(currNode).getText()); + tblNameOrPattern = PlanUtils.stripQuotes(ast.getChild(currNode).getChild(0).getText()); } else { // TOK_FROM subtree Tree fromNode = ast.getChild(currNode); @@ -152,8 +159,6 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { // move to the next child in FROM tree numChild++; } - // FROM node is always the last - break; } // move to the next root node currNode++; @@ -176,7 +181,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer { ErrorMsg.INVALID_PATH.getMsg(ast), maxEventLimit, ctx.getResFile().toUri().toString() - ), conf); + ), conf, true); rootTasks.add(replDumpWorkTask); if (dbNameOrPattern != null) { for (String dbName : Utils.matchesDb(db, dbNameOrPattern)) { http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java index 3f3c4c8..5844f3d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/PartitionExport.java @@ -1,19 +1,19 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.ql.parse.repl.dump; http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java index a44f98f..e1cea22 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java @@ -1,19 +1,19 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.ql.parse.repl.dump; @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.hooks.ReadEntity; import org.apache.hadoop.hive.ql.hooks.WriteEntity; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; @@ -34,7 +35,6 @@ import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec; import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; -import org.apache.hadoop.hive.ql.parse.repl.dump.TableExport.AuthEntities; import org.apache.hadoop.hive.ql.parse.repl.dump.io.FileOperations; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,16 +60,16 @@ public class TableExport { private final HiveConf conf; private final Paths paths; - public TableExport(Paths paths, TableSpec tableSpec, - ReplicationSpec replicationSpec, Hive db, String distCpDoAsUser, HiveConf conf) - throws SemanticException { + public TableExport(Paths paths, TableSpec tableSpec, ReplicationSpec replicationSpec, Hive db, + String distCpDoAsUser, HiveConf conf) { this.tableSpec = (tableSpec != null && tableSpec.tableHandle.isTemporary() && replicationSpec.isInReplicationScope()) ? null : tableSpec; this.replicationSpec = replicationSpec; - if (this.tableSpec != null && this.tableSpec.tableHandle.isView()) { + if (conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY) || (this.tableSpec != null + && this.tableSpec.tableHandle.isView())) { this.replicationSpec.setIsMetadataOnly(true); } this.db = db; @@ -78,20 +78,19 @@ public class TableExport { this.paths = paths; } - public void write() throws SemanticException { + public boolean write() throws SemanticException { if (tableSpec == null) { writeMetaData(null); + return true; } else if (shouldExport()) { - //first we should get the correct replication spec before doing metadata/data export - if (tableSpec.tableHandle.isView()) { - replicationSpec.setIsMetadataOnly(true); - } PartitionIterable withPartitions = getPartitions(); writeMetaData(withPartitions); if (!replicationSpec.isMetadataOnly()) { writeData(withPartitions); } + return true; } + return false; } private PartitionIterable getPartitions() throws SemanticException { @@ -130,7 +129,8 @@ public class TableExport { paths.metaDataExportFile(), tableSpec == null ? null : tableSpec.tableHandle, partitions, - replicationSpec); + replicationSpec, + conf); logger.debug("_metadata file written into " + paths.metaDataExportFile().toString()); } catch (Exception e) { // the path used above should not be used on a second try as each dump request is written to a unique location. @@ -159,8 +159,12 @@ public class TableExport { } } - private boolean shouldExport() throws SemanticException { - return EximUtil.shouldExportTable(replicationSpec, tableSpec.tableHandle); + private boolean shouldExport() { + if (conf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_INCLUDE_ACID_TABLES) + && AcidUtils.isAcidTable(tableSpec.tableHandle)) { + return true; + } + return Utils.shouldReplicate(replicationSpec, tableSpec.tableHandle, conf); } /** @@ -172,7 +176,7 @@ public class TableExport { private final HiveConf conf; private final Path exportRootDir; private final FileSystem exportFileSystem; - private boolean writeData = true; + private boolean writeData; public Paths(String astRepresentationForErrorMsg, Path dbRoot, String tblName, HiveConf conf, boolean shouldWriteData) throws SemanticException { http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java index 97f0e0b..f880913 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,9 +21,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.io.IOUtils; @@ -152,4 +156,46 @@ public class Utils { } return false; } + + /** + * validates if a table can be exported, similar to EximUtil.shouldExport with few replication + * specific checks. + */ + public static Boolean shouldReplicate(ReplicationSpec replicationSpec, Table tableHandle, + HiveConf hiveConf) { + if (replicationSpec == null) { + replicationSpec = new ReplicationSpec(); + } + + if (replicationSpec.isNoop() || tableHandle == null) { + return false; + } + + if (tableHandle.isNonNative()) { + return false; + } + + if (replicationSpec.isInReplicationScope()) { + boolean isAcidTable = AcidUtils.isAcidTable(tableHandle); + if (isAcidTable) { + return hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_INCLUDE_ACID_TABLES); + } + return !tableHandle.isTemporary(); + } + return true; + } + + public static boolean shouldReplicate(NotificationEvent tableForEvent, + ReplicationSpec replicationSpec, Hive db, HiveConf hiveConf) { + Table table; + try { + table = db.getTable(tableForEvent.getDbName(), tableForEvent.getTableName()); + } catch (HiveException e) { + LOG.info( + "error while getting table info for" + tableForEvent.getDbName() + "." + tableForEvent + .getTableName(), e); + return false; + } + return shouldReplicate(replicationSpec, table, hiveConf); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AbstractConstraintEventHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AbstractConstraintEventHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AbstractConstraintEventHandler.java new file mode 100644 index 0000000..3ed005c --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AbstractConstraintEventHandler.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse.repl.dump.events; + +import org.apache.hadoop.hive.metastore.api.NotificationEvent; +import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; + +abstract class AbstractConstraintEventHandler extends AbstractEventHandler { + AbstractConstraintEventHandler(NotificationEvent event) { + super(event); + } + + boolean shouldReplicate(Context withinContext) { + return Utils.shouldReplicate( + event, + withinContext.replicationSpec, + withinContext.db, + withinContext.hiveConf + ); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java index d0cbd4a..8fdf2f1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddForeignKeyHandler.java @@ -18,28 +18,23 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; -public class AddForeignKeyHandler extends AbstractEventHandler { +public class AddForeignKeyHandler extends AbstractConstraintEventHandler { AddForeignKeyHandler(NotificationEvent event) { super(event); } @Override public void handle(Context withinContext) throws Exception { - LOG.info("Processing#{} ADD_FOREIGNKEY_MESSAGE message : {}", fromEventId(), event.getMessage()); - - if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) - { - return; + LOG.debug("Processing#{} ADD_FOREIGNKEY_MESSAGE message : {}", fromEventId(), + event.getMessage()); + if (shouldReplicate(withinContext)) { + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(event.getMessage()); + dmd.write(); } - - DumpMetaData dmd = withinContext.createDmd(this); - dmd.setPayload(event.getMessage()); - dmd.write(); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java index aa7f4ef..335d4e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddNotNullConstraintHandler.java @@ -18,28 +18,24 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; -public class AddNotNullConstraintHandler extends AbstractEventHandler { +public class AddNotNullConstraintHandler extends AbstractConstraintEventHandler { AddNotNullConstraintHandler(NotificationEvent event) { super(event); } @Override public void handle(Context withinContext) throws Exception { - LOG.info("Processing#{} ADD_NOTNULLCONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage()); + LOG.debug("Processing#{} ADD_NOTNULLCONSTRAINT_MESSAGE message : {}", fromEventId(), + event.getMessage()); - if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) - { - return; + if (shouldReplicate(withinContext)) { + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(event.getMessage()); + dmd.write(); } - - DumpMetaData dmd = withinContext.createDmd(this); - dmd.setPayload(event.getMessage()); - dmd.write(); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java index def8384..cf15905 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPartitionHandler.java @@ -17,8 +17,6 @@ */ package org.apache.hadoop.hive.ql.parse.repl.dump.events; -import com.google.common.base.Function; -import com.google.common.collect.Iterables; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.NotificationEvent; @@ -28,14 +26,15 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.EximUtil; +import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; -import javax.annotation.Nullable; import java.io.BufferedWriter; import java.io.IOException; import java.io.OutputStreamWriter; import java.util.Iterator; - -import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; class AddPartitionHandler extends AbstractEventHandler { protected AddPartitionHandler(NotificationEvent notificationEvent) { @@ -54,7 +53,7 @@ class AddPartitionHandler extends AbstractEventHandler { } final Table qlMdTable = new Table(tobj); - if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) { + if (!Utils.shouldReplicate(withinContext.replicationSpec, qlMdTable, withinContext.hiveConf)) { return; } @@ -64,23 +63,17 @@ class AddPartitionHandler extends AbstractEventHandler { return; } - Iterable qlPtns = Iterables.transform( - ptns, - new Function() { - @Nullable - @Override - public Partition apply(@Nullable org.apache.hadoop.hive.metastore.api.Partition input) { - if (input == null) { - return null; - } - try { - return new Partition(qlMdTable, input); - } catch (HiveException e) { - throw new IllegalArgumentException(e); - } + Iterable qlPtns = StreamSupport.stream(ptns.spliterator(), true).map( + input -> { + if (input == null) { + return null; } - } - ); + try { + return new Partition(qlMdTable, input); + } catch (HiveException e) { + throw new IllegalArgumentException(e); + } + }).collect(Collectors.toList()); Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME); EximUtil.createExportDump( @@ -88,7 +81,8 @@ class AddPartitionHandler extends AbstractEventHandler { metaDataPath, qlMdTable, qlPtns, - withinContext.replicationSpec); + withinContext.replicationSpec, + withinContext.hiveConf); Iterator partitionFilesIter = apm.getPartitionFilesIter().iterator(); for (Partition qlPtn : qlPtns) { http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java index 344fac9..cf45c68 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddPrimaryKeyHandler.java @@ -18,28 +18,24 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; -public class AddPrimaryKeyHandler extends AbstractEventHandler { +public class AddPrimaryKeyHandler extends AbstractConstraintEventHandler { AddPrimaryKeyHandler(NotificationEvent event) { super(event); } @Override public void handle(Context withinContext) throws Exception { - LOG.info("Processing#{} ADD_PRIMARYKEY_MESSAGE message : {}", fromEventId(), event.getMessage()); + LOG.debug("Processing#{} ADD_PRIMARYKEY_MESSAGE message : {}", fromEventId(), + event.getMessage()); - if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) - { - return; + if (shouldReplicate(withinContext)) { + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(event.getMessage()); + dmd.write(); } - - DumpMetaData dmd = withinContext.createDmd(this); - dmd.setPayload(event.getMessage()); - dmd.write(); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java index 4cc75a7..58835a0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AddUniqueConstraintHandler.java @@ -18,28 +18,24 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; -public class AddUniqueConstraintHandler extends AbstractEventHandler { +public class AddUniqueConstraintHandler extends AbstractConstraintEventHandler { AddUniqueConstraintHandler(NotificationEvent event) { super(event); } @Override public void handle(Context withinContext) throws Exception { - LOG.info("Processing#{} ADD_UNIQUECONSTRAINT_MESSAGE message : {}", fromEventId(), event.getMessage()); + LOG.debug("Processing#{} ADD_UNIQUECONSTRAINT_MESSAGE message : {}", fromEventId(), + event.getMessage()); - if (!EximUtil.tryValidateShouldExportTable(withinContext.db, event.getDbName(), event.getTableName(), withinContext.replicationSpec)) - { - return; + if (shouldReplicate(withinContext)) { + DumpMetaData dmd = withinContext.createDmd(this); + dmd.setPayload(event.getMessage()); + dmd.write(); } - - DumpMetaData dmd = withinContext.createDmd(this); - dmd.setPayload(event.getMessage()); - dmd.write(); } @Override http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java index 58df665..cde4eed 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterPartitionHandler.java @@ -30,6 +30,7 @@ import java.util.List; import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; class AlterPartitionHandler extends AbstractEventHandler { @@ -88,7 +89,7 @@ class AlterPartitionHandler extends AbstractEventHandler { LOG.info("Processing#{} ALTER_PARTITION message : {}", fromEventId(), event.getMessage()); Table qlMdTable = new Table(tableObject); - if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) { + if (!Utils.shouldReplicate(withinContext.replicationSpec, qlMdTable, withinContext.hiveConf)) { return; } @@ -102,7 +103,8 @@ class AlterPartitionHandler extends AbstractEventHandler { metaDataPath, qlMdTable, partitions, - withinContext.replicationSpec); + withinContext.replicationSpec, + withinContext.hiveConf); } DumpMetaData dmd = withinContext.createDmd(this); dmd.setPayload(event.getMessage()); http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java index 4e3ce0e..5f582b3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/AlterTableHandler.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; class AlterTableHandler extends AbstractEventHandler { @@ -80,7 +81,8 @@ class AlterTableHandler extends AbstractEventHandler { LOG.info("Processing#{} ALTER_TABLE message : {}", fromEventId(), event.getMessage()); Table qlMdTableBefore = new Table(before); - if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTableBefore)) { + if (!Utils + .shouldReplicate(withinContext.replicationSpec, qlMdTableBefore, withinContext.hiveConf)) { return; } @@ -89,11 +91,12 @@ class AlterTableHandler extends AbstractEventHandler { Table qlMdTableAfter = new Table(after); Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME); EximUtil.createExportDump( - metaDataPath.getFileSystem(withinContext.hiveConf), - metaDataPath, - qlMdTableAfter, - null, - withinContext.replicationSpec); + metaDataPath.getFileSystem(withinContext.hiveConf), + metaDataPath, + qlMdTableAfter, + null, + withinContext.replicationSpec, + withinContext.hiveConf); } DumpMetaData dmd = withinContext.createDmd(this); http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java index ef6f340..3804396 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CreateTableHandler.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; import java.io.BufferedWriter; import java.io.IOException; @@ -48,7 +49,7 @@ class CreateTableHandler extends AbstractEventHandler { Table qlMdTable = new Table(tobj); - if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) { + if (!Utils.shouldReplicate(withinContext.replicationSpec, qlMdTable, withinContext.hiveConf)) { return; } @@ -62,7 +63,8 @@ class CreateTableHandler extends AbstractEventHandler { metaDataPath, qlMdTable, null, - withinContext.replicationSpec); + withinContext.replicationSpec, + withinContext.hiveConf); Path dataPath = new Path(withinContext.eventRoot, "data"); Iterable files = ctm.getFiles(); http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java index df852a3..5ac3af0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/InsertHandler.java @@ -19,11 +19,13 @@ package org.apache.hadoop.hive.ql.parse.repl.dump.events; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.metastore.messaging.InsertMessage; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.repl.DumpType; +import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData; import java.io.BufferedWriter; @@ -41,10 +43,13 @@ class InsertHandler extends AbstractEventHandler { @Override public void handle(Context withinContext) throws Exception { + if (withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY)) { + return; + } InsertMessage insertMsg = deserializer.getInsertMessage(event.getMessage()); org.apache.hadoop.hive.ql.metadata.Table qlMdTable = tableObject(insertMsg); - if (!EximUtil.shouldExportTable(withinContext.replicationSpec, qlMdTable)) { + if (!Utils.shouldReplicate(withinContext.replicationSpec, qlMdTable, withinContext.hiveConf)) { return; } @@ -58,7 +63,8 @@ class InsertHandler extends AbstractEventHandler { withinContext.replicationSpec.setIsReplace(insertMsg.isReplace()); EximUtil.createExportDump(metaDataPath.getFileSystem(withinContext.hiveConf), metaDataPath, qlMdTable, qlPtns, - withinContext.replicationSpec); + withinContext.replicationSpec, + withinContext.hiveConf); Iterable files = insertMsg.getFiles(); if (files != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java index 4642012..866d351 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java @@ -1,19 +1,19 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.ql.parse.repl.dump.io; http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java index c3a70cc..143808b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java @@ -17,13 +17,14 @@ */ package org.apache.hadoop.hive.ql.parse.repl.dump.io; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.metadata.Partition; -import org.apache.hadoop.hive.ql.parse.EximUtil; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.repl.dump.Utils; import org.apache.thrift.TException; import org.apache.thrift.TSerializer; import org.apache.thrift.protocol.TJSONProtocol; @@ -35,17 +36,19 @@ public class TableSerializer implements JsonWriter.Serializer { public static final String FIELD_NAME = "table"; private final org.apache.hadoop.hive.ql.metadata.Table tableHandle; private final Iterable partitions; + private final HiveConf hiveConf; public TableSerializer(org.apache.hadoop.hive.ql.metadata.Table tableHandle, - Iterable partitions) { + Iterable partitions, HiveConf hiveConf) { this.tableHandle = tableHandle; this.partitions = partitions; + this.hiveConf = hiveConf; } @Override public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider) throws SemanticException, IOException { - if (!EximUtil.shouldExportTable(additionalPropertiesProvider, tableHandle)) { + if (!Utils.shouldReplicate(additionalPropertiesProvider, tableHandle, hiveConf)) { return; } @@ -62,8 +65,7 @@ public class TableSerializer implements JsonWriter.Serializer { } } - private Table addPropertiesToTable(Table table, ReplicationSpec additionalPropertiesProvider) - throws SemanticException, IOException { + private Table addPropertiesToTable(Table table, ReplicationSpec additionalPropertiesProvider) { if (additionalPropertiesProvider.isInReplicationScope()) { // Current replication state must be set on the Table object only for bootstrap dump. // Event replication State will be null in case of bootstrap dump. http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/java/org/apache/hadoop/hive/ql/plan/ExportWork.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExportWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExportWork.java index ac723b6..9093f48 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExportWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExportWork.java @@ -1,19 +1,19 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.ql.plan; http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/bootstrap/AddDependencyToLeavesTest.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/bootstrap/AddDependencyToLeavesTest.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/bootstrap/AddDependencyToLeavesTest.java index a807483..aa24d29 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/bootstrap/AddDependencyToLeavesTest.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/repl/bootstrap/AddDependencyToLeavesTest.java @@ -1,19 +1,19 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.ql.exec.repl.bootstrap; http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/test/org/apache/hadoop/hive/ql/exec/util/DAGTraversalTest.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/util/DAGTraversalTest.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/util/DAGTraversalTest.java index 4bce6bc..6dcecde 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/util/DAGTraversalTest.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/util/DAGTraversalTest.java @@ -1,21 +1,20 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ - package org.apache.hadoop.hive.ql.exec.util; import org.apache.hadoop.hive.ql.exec.Task; http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java index 3305998..8de4844 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestReplicationSemanticAnalyzer.java @@ -1,304 +1,257 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.ql.parse; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.Context; -import org.apache.hadoop.hive.ql.QueryState; -import org.apache.hadoop.hive.ql.exec.FetchTask; -import org.apache.hadoop.hive.ql.exec.Task; -import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; -import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat; -import org.apache.hadoop.hive.ql.metadata.Hive; -import org.apache.hadoop.hive.ql.metadata.HiveException; -import org.apache.hadoop.hive.ql.metadata.Table; -import org.apache.hadoop.hive.ql.session.SessionState; -import org.junit.AfterClass; -import org.junit.BeforeClass; import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +@RunWith(Enclosed.class) public class TestReplicationSemanticAnalyzer { - private static QueryState queryState; - static HiveConf conf; - private static String defaultDB = "default"; - private static String tblName = "testReplSA"; - private static ArrayList cols = new ArrayList(Arrays.asList("col1", "col2")); - - @BeforeClass - public static void initialize() throws HiveException { - queryState = - new QueryState.Builder().withHiveConf(new HiveConf(SemanticAnalyzer.class)).build(); - conf = queryState.getConf(); - conf.set("hive.security.authorization.manager", ""); - SessionState.start(conf); - Hive hiveDb = Hive.get(conf); - hiveDb.createTable(defaultDB + "." + tblName, cols, null, OrcInputFormat.class, OrcOutputFormat.class); - Table t = hiveDb.getTable(tblName); - } + private static ParseDriver driver = new ParseDriver(); - @AfterClass - public static void teardown() throws HiveException { + private static ASTNode parse(String command) throws ParseException { + return (ASTNode) driver.parse(command).getChild(0); } - @Test - public void testReplDumpParse() throws Exception { - ParseDriver pd = new ParseDriver(); - String fromEventId = "100"; - String toEventId = "200"; - String maxEventLimit = "50"; - ASTNode root; - ASTNode child; - - String query = "repl dump " + defaultDB; - root = (ASTNode) pd.parse(query).getChild(0); - assertEquals(root.getText(), "TOK_REPL_DUMP"); - assertEquals(root.getChildCount(), 1); - - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), defaultDB); - assertEquals(child.getChildCount(), 0); - - query = "repl dump " + defaultDB + "." + tblName; - root = (ASTNode) pd.parse(query).getChild(0); - assertEquals(root.getChildCount(), 2); - - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), defaultDB); - assertEquals(child.getChildCount(), 0); - - child = (ASTNode) root.getChild(1); - assertEquals(child.getText(), tblName); - assertEquals(child.getChildCount(), 0); - - query = "repl dump " + defaultDB + "." + tblName + " from " + fromEventId; - root = (ASTNode) pd.parse(query).getChild(0); - assertEquals(root.getChildCount(), 3); - - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), defaultDB); - assertEquals(child.getChildCount(), 0); - - child = (ASTNode) root.getChild(1); - assertEquals(child.getText(), tblName); - assertEquals(child.getChildCount(), 0); - - root = (ASTNode) root.getChild(2); - assertEquals(root.getText(), "TOK_FROM"); - assertEquals(root.getChildCount(), 1); - - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), fromEventId); - assertEquals(child.getChildCount(), 0); - - query = "repl dump " + defaultDB + "." + tblName + " from " + fromEventId + " to " + toEventId; - - root = (ASTNode) pd.parse(query).getChild(0); - assertEquals(root.getChildCount(), 3); + private static void assertWithClause(ASTNode root, int replConfigIndex) { + ASTNode replConfig = (ASTNode) root.getChild(replConfigIndex); + assertEquals("TOK_REPL_CONFIG", replConfig.getText()); + assertEquals(1, replConfig.getChildCount()); + ASTNode replConfigList = (ASTNode) replConfig.getChild(0); + assertEquals("TOK_REPL_CONFIG_LIST", replConfigList.getText()); + assertEquals(2, replConfigList.getChildCount()); - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), defaultDB); - assertEquals(child.getChildCount(), 0); - - child = (ASTNode) root.getChild(1); - assertEquals(child.getText(), tblName); - assertEquals(child.getChildCount(), 0); - - root = (ASTNode) root.getChild(2); - assertEquals(root.getText(), "TOK_FROM"); - assertEquals(root.getChildCount(), 3); - - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), fromEventId); - assertEquals(child.getChildCount(), 0); - - child = (ASTNode) root.getChild(1); - assertEquals(child.getText(), "TOK_TO"); - assertEquals(child.getChildCount(), 0); - - child = (ASTNode) root.getChild(2); - assertEquals(child.getText(), toEventId); - assertEquals(child.getChildCount(), 0); - - query = - "repl dump " + defaultDB + "." + tblName + " from " + fromEventId + " to " + toEventId - + " limit " + maxEventLimit; - - root = (ASTNode) pd.parse(query).getChild(0); - assertEquals(root.getChildCount(), 3); - - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), defaultDB); - assertEquals(child.getChildCount(), 0); - - child = (ASTNode) root.getChild(1); - assertEquals(child.getText(), tblName); - assertEquals(child.getChildCount(), 0); + assertConfig(replConfigList, 0, "'key.1'", "'value.1'"); + assertConfig(replConfigList, 1, "'key.2'", "'value.2'"); + } - root = (ASTNode) root.getChild(2); - assertEquals(root.getText(), "TOK_FROM"); - assertEquals(root.getChildCount(), 5); + private static void assertConfig(ASTNode replConfigList, int atIndex, String expectedKey, + String expectedValue) { + ASTNode configOne = (ASTNode) replConfigList.getChild(atIndex); + assertEquals("TOK_TABLEPROPERTY", configOne.getText()); + assertEquals(2, configOne.getChildCount()); + assertEquals(expectedKey, configOne.getChild(0).getText()); + assertEquals(expectedValue, configOne.getChild(1).getText()); + } - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), fromEventId); - assertEquals(child.getChildCount(), 0); - - child = (ASTNode) root.getChild(1); - assertEquals(child.getText(), "TOK_TO"); - assertEquals(child.getChildCount(), 0); + private static void assertToEventId(ASTNode fromClauseRootNode) { + ASTNode child = (ASTNode) fromClauseRootNode.getChild(1); + assertEquals("TOK_TO", child.getText()); + assertEquals(0, child.getChildCount()); - child = (ASTNode) root.getChild(2); - assertEquals(child.getText(), toEventId); - assertEquals(child.getChildCount(), 0); + child = (ASTNode) fromClauseRootNode.getChild(2); + assertEquals("200", child.getText()); + assertEquals(0, child.getChildCount()); + } - child = (ASTNode) root.getChild(3); - assertEquals(child.getText(), "TOK_LIMIT"); - assertEquals(child.getChildCount(), 0); + private static ASTNode assertFromEvent(final int expectedNumberOfChildren, ASTNode root) { + ASTNode child = (ASTNode) root.getChild(2); + assertEquals("TOK_FROM", child.getText()); + assertEquals(child.getChildCount(), expectedNumberOfChildren); - child = (ASTNode) root.getChild(4); - assertEquals(child.getText(), maxEventLimit); - assertEquals(child.getChildCount(), 0); + ASTNode fromClauseChild = (ASTNode) child.getChild(0); + assertEquals("100", fromClauseChild.getText()); + assertEquals(0, fromClauseChild.getChildCount()); + return child; } - @Test - public void testReplLoadParse() throws Exception { - // FileSystem fs = FileSystem.get(conf); - ParseDriver pd = new ParseDriver(); - ASTNode root; - ASTNode child; - ASTNode subChild; - ASTNode configNode; - String replRoot = conf.getVar(HiveConf.ConfVars.REPLDIR); - Path dumpRoot = new Path(replRoot, "next"); - System.out.println(replRoot); - System.out.println(dumpRoot); - String newDB = "default_bak"; - String newDB2= "default_bak_2"; - - String query = "repl load from '" + dumpRoot.toString() + "'"; - root = (ASTNode) pd.parse(query).getChild(0); - assertEquals(root.getText(), "TOK_REPL_LOAD"); - assertEquals(root.getChildCount(), 1); - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), "'" + dumpRoot.toString() + "'"); - assertEquals(child.getChildCount(), 0); - - query = "repl load " + newDB + " from '" + dumpRoot.toString() + "'"; - root = (ASTNode) pd.parse(query).getChild(0); - assertEquals(root.getText(), "TOK_REPL_LOAD"); - assertEquals(root.getChildCount(), 2); - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), "'" + dumpRoot.toString() + "'"); - assertEquals(child.getChildCount(), 0); - child = (ASTNode) root.getChild(1); - assertEquals(child.getText(), "TOK_DBNAME"); - assertEquals(child.getChildCount(), 1); - subChild = (ASTNode) child.getChild(0); - assertEquals(subChild.getText(), newDB); - assertEquals(subChild.getChildCount(), 0); - - query = "repl load " + newDB2 + " from '" + dumpRoot.toString() - + "' with ('mapred.job.queue.name'='repl','hive.repl.approx.max.load.tasks'='100')"; - root = (ASTNode) pd.parse(query).getChild(0); - assertEquals(root.getText(), "TOK_REPL_LOAD"); - assertEquals(root.getChildCount(), 3); - child = (ASTNode) root.getChild(0); - assertEquals(child.getText(), "'" + dumpRoot.toString() + "'"); - assertEquals(child.getChildCount(), 0); - child = (ASTNode) root.getChild(1); - assertEquals(child.getText(), "TOK_DBNAME"); - assertEquals(child.getChildCount(), 1); - subChild = (ASTNode) child.getChild(0); - assertEquals(subChild.getText(), newDB2); - assertEquals(subChild.getChildCount(), 0); - child = (ASTNode) root.getChild(2); - assertEquals(child.getText(), "TOK_REPL_CONFIG"); - assertEquals(child.getChildCount(), 1); - subChild = (ASTNode) child.getChild(0); - assertEquals(subChild.getText(), "TOK_REPL_CONFIG_LIST"); - assertEquals(subChild.getChildCount(), 2); - configNode = (ASTNode) subChild.getChild(0); - assertEquals(configNode.getText(), "TOK_TABLEPROPERTY"); - assertEquals(configNode.getChildCount(), 2); - assertEquals(configNode.getChild(0).getText(), "'mapred.job.queue.name'"); - assertEquals(configNode.getChild(1).getText(), "'repl'"); - configNode = (ASTNode) subChild.getChild(1); - assertEquals(configNode.getText(), "TOK_TABLEPROPERTY"); - assertEquals(configNode.getChildCount(), 2); - assertEquals(configNode.getChild(0).getText(), "'hive.repl.approx.max.load.tasks'"); - assertEquals(configNode.getChild(1).getText(), "'100'"); + private static void assertTableName(ASTNode root) { + ASTNode child = (ASTNode) root.getChild(1); + assertEquals("TOK_TABNAME", child.getText()); + assertEquals(1, child.getChildCount()); + assertEquals("test_table", child.getChild(0).getText()); } - //@Test - public void testReplLoadAnalyze() throws Exception { - ParseDriver pd = new ParseDriver(); - ASTNode root; - String replRoot = conf.getVar(HiveConf.ConfVars.REPLDIR); - FileSystem fs = FileSystem.get(conf); - Path dumpRoot = new Path(replRoot, "next"); - System.out.println(replRoot); - System.out.println(dumpRoot); - String newDB = "default_bak"; - - // First create a dump - String query = "repl dump " + defaultDB; - root = (ASTNode) pd.parse(query).getChild(0); - ReplicationSemanticAnalyzer rs = (ReplicationSemanticAnalyzer) SemanticAnalyzerFactory.get(queryState, root); - rs.analyze(root, new Context(conf)); - - // Then analyze load - query = "repl load from '" + dumpRoot.toString() + "'"; - root = (ASTNode) pd.parse(query).getChild(0); - rs = (ReplicationSemanticAnalyzer) SemanticAnalyzerFactory.get(queryState, root); - rs.analyze(root, new Context(conf)); - List> roots = rs.getRootTasks(); - assertEquals(1, roots.size()); - - query = "repl load " + newDB + " from '" + dumpRoot.toString() + "'"; - root = (ASTNode) pd.parse(query).getChild(0); - rs = (ReplicationSemanticAnalyzer) SemanticAnalyzerFactory.get(queryState, root); - rs.analyze(root, new Context(conf)); - roots = rs.getRootTasks(); - assertEquals(1, roots.size()); + private static void assertDatabase(final int expectedNumberOfChildren, ASTNode root) { + assertEquals("TOK_REPL_DUMP", root.getText()); + assertEquals(expectedNumberOfChildren, root.getChildCount()); + ASTNode child = (ASTNode) root.getChild(0); + assertEquals("testDb", child.getText()); + assertEquals(0, child.getChildCount()); } - @Test - public void testReplStatusAnalyze() throws Exception { - ParseDriver pd = new ParseDriver(); - ASTNode root; + public static class ReplDump { + + @Test + public void parseDb() throws ParseException { + ASTNode root = parse("repl dump testDb"); + assertDatabase(1, root); + } + + @Test + public void parseTableName() throws ParseException { + ASTNode root = parse("repl dump testDb.test_table"); + assertDatabase(2, root); + assertTableName(root); + } + + @Test + public void parseFromEventId() throws ParseException { + ASTNode root = parse("repl dump testDb.test_table from 100"); + assertDatabase(3, root); + assertTableName(root); + assertFromEvent(1, root); + } + + @Test + public void parseToEventId() throws ParseException { + ASTNode root = parse("repl dump testDb.test_table from 100 to 200"); + assertDatabase(3, root); + assertTableName(root); + ASTNode fromClauseRootNode = assertFromEvent(3, root); + assertToEventId(fromClauseRootNode); + } + + @Test + public void parseLimit() throws ParseException { + ASTNode root = parse("repl dump testDb.test_table from 100 to 200 limit 10"); + assertDatabase(3, root); + assertTableName(root); + ASTNode fromClauseRootNode = assertFromEvent(5, root); + assertToEventId(fromClauseRootNode); + + ASTNode child = (ASTNode) fromClauseRootNode.getChild(3); + assertEquals("TOK_LIMIT", child.getText()); + assertEquals(0, child.getChildCount()); + + child = (ASTNode) fromClauseRootNode.getChild(4); + assertEquals("10", child.getText()); + assertEquals(0, child.getChildCount()); + } + } - // Repl status command - String query = "repl status " + defaultDB; - root = (ASTNode) pd.parse(query).getChild(0); - ReplicationSemanticAnalyzer rs = (ReplicationSemanticAnalyzer) SemanticAnalyzerFactory.get(queryState, root); - rs.analyze(root, new Context(conf)); + public static class ReplDumpWithClause { + + @Test + public void parseDb() throws ParseException { + ASTNode root = parse("repl dump testDb with ('key.1'='value.1','key.2'='value.2')"); + assertDatabase(2, root); + assertWithClause(root, 1); + } + + @Test + public void parseTableName() throws ParseException { + ASTNode root = + parse("repl dump testDb.test_table with ('key.1'='value.1','key.2'='value.2')"); + assertDatabase(3, root); + assertTableName(root); + assertWithClause(root, 2); + } + + @Test + public void parseFromEventId() throws ParseException { + ASTNode root = parse("repl dump testDb.test_table from 100 " + + "with ('key.1'='value.1','key.2'='value.2')"); + assertDatabase(4, root); + assertTableName(root); + assertFromEvent(1, root); + assertWithClause(root, 3); + } + + @Test + public void parseToEventId() throws ParseException { + ASTNode root = parse("repl dump testDb.test_table from 100 to 200 " + + "with ('key.1'='value.1','key.2'='value.2')"); + assertDatabase(4, root); + assertTableName(root); + ASTNode fromClauseRootNode = assertFromEvent(3, root); + assertToEventId(fromClauseRootNode); + assertWithClause(root, 3); + } + + @Test + public void parseLimit() throws ParseException { + ASTNode root = parse("repl dump testDb.test_table from 100 to 200 limit 10 " + + "with ('key.1'='value.1','key.2'='value.2')"); + assertDatabase(4, root); + assertTableName(root); + ASTNode fromClauseRootNode = assertFromEvent(5, root); + assertToEventId(fromClauseRootNode); + assertWithClause(root, 3); + + ASTNode child = (ASTNode) fromClauseRootNode.getChild(3); + assertEquals("TOK_LIMIT", child.getText()); + assertEquals(0, child.getChildCount()); + + child = (ASTNode) fromClauseRootNode.getChild(4); + assertEquals("10", child.getText()); + assertEquals(0, child.getChildCount()); + } + } - FetchTask fetchTask = rs.getFetchTask(); - assertNotNull(fetchTask); + public static class ReplLoad { + + @Test + public void parseFromLocation() throws ParseException { + ASTNode root = parse("repl load from '/some/location/in/hdfs/'"); + assertFromLocation(1, root); + } + + @Test + public void parseTargetDbName() throws ParseException { + ASTNode root = parse("repl load targetTestDbName from '/some/location/in/hdfs/'"); + assertFromLocation(2, root); + assertTargetDatabaseName(root); + } + + @Test + public void parseWithClause() throws ParseException { + ASTNode root = parse("repl load targetTestDbName from '/some/location/in/hdfs/'" + + " with ('mapred.job.queue.name'='repl','hive.repl.approx.max.load.tasks'='100')"); + assertFromLocation(3, root); + assertTargetDatabaseName(root); + + ASTNode child = (ASTNode) root.getChild(2); + assertEquals("TOK_REPL_CONFIG", child.getText()); + assertEquals(1, child.getChildCount()); + child = (ASTNode) child.getChild(0); + assertEquals("TOK_REPL_CONFIG_LIST", child.getText()); + assertEquals(2, child.getChildCount()); + ASTNode configNode = (ASTNode) child.getChild(0); + assertEquals("TOK_TABLEPROPERTY", configNode.getText()); + assertEquals(2, configNode.getChildCount()); + assertEquals("'mapred.job.queue.name'", configNode.getChild(0).getText()); + assertEquals("'repl'", configNode.getChild(1).getText()); + configNode = (ASTNode) child.getChild(1); + assertEquals("TOK_TABLEPROPERTY", configNode.getText()); + assertEquals(2, configNode.getChildCount()); + assertEquals("'hive.repl.approx.max.load.tasks'", configNode.getChild(0).getText()); + assertEquals("'100'", configNode.getChild(1).getText()); + } + + private void assertFromLocation(final int expectedNumberOfChildren, ASTNode root) { + assertEquals("TOK_REPL_LOAD", root.getText()); + assertEquals(expectedNumberOfChildren, root.getChildCount()); + ASTNode child = (ASTNode) root.getChild(0); + assertEquals("'/some/location/in/hdfs/'", child.getText()); + assertEquals(0, child.getChildCount()); + } + + private void assertTargetDatabaseName(ASTNode root) { + ASTNode child = (ASTNode) root.getChild(1); + assertEquals("TOK_DBNAME", child.getText()); + assertEquals(1, child.getChildCount()); + child = (ASTNode) child.getChild(0); + assertEquals("targetTestDbName", child.getText()); + assertEquals(0, child.getChildCount()); + } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/fd4e222d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java index 5b4d4bd..b23a6d7 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java @@ -1,19 +1,19 @@ /* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.hive.metastore.tools;