hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject [1/2] hive git commit: HIVE-16267 : Enable bootstrap function metadata to be loaded in repl load (Anishek Agarwal, reviewed by Sushanth Sowmyan)
Date Thu, 04 May 2017 09:49:31 GMT
Repository: hive
Updated Branches:
  refs/heads/master f56abb405 -> 9e9356b5e


http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
new file mode 100644
index 0000000..077d39b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/PartitionSerializer.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.io;
+
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TJSONProtocol;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class PartitionSerializer implements JsonWriter.Serializer {
+  public static final String FIELD_NAME="partitions";
+  private Partition partition;
+
+  PartitionSerializer(Partition partition) {
+    this.partition = partition;
+  }
+
+  @Override
+  public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+      throws SemanticException, IOException {
+    TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+    try {
+      if (additionalPropertiesProvider.isInReplicationScope()) {
+        partition.putToParameters(
+            ReplicationSpec.KEY.CURR_STATE_ID.toString(),
+            additionalPropertiesProvider.getCurrentReplicationState());
+        if (isPartitionExternal()) {
+          // Replication destination will not be external
+          partition.putToParameters("EXTERNAL", "FALSE");
+        }
+      }
+      writer.jsonGenerator.writeString(serializer.toString(partition, UTF_8));
+      writer.jsonGenerator.flush();
+    } catch (TException e) {
+      throw new SemanticException(ErrorMsg.ERROR_SERIALIZE_METASTORE.getMsg(), e);
+    }
+  }
+
+  private boolean isPartitionExternal() {
+    Map<String, String> params = partition.getParameters();
+    return params.containsKey("EXTERNAL")
+        && params.get("EXTERNAL").equalsIgnoreCase("TRUE");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ReplicationSpecSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ReplicationSpecSerializer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ReplicationSpecSerializer.java
new file mode 100644
index 0000000..3a92e8a
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/ReplicationSpecSerializer.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.io;
+
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+import java.io.IOException;
+
+public class ReplicationSpecSerializer implements JsonWriter.Serializer {
+  @Override
+  public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+      throws SemanticException, IOException {
+    for (ReplicationSpec.KEY key : ReplicationSpec.KEY.values()) {
+      String value = additionalPropertiesProvider.get(key);
+      if (value != null) {
+        writer.jsonGenerator.writeStringField(key.toString(), value);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
new file mode 100644
index 0000000..948cb39
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/TableSerializer.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.io;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TJSONProtocol;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class TableSerializer implements JsonWriter.Serializer {
+  public static final String FIELD_NAME = "table";
+  private final org.apache.hadoop.hive.ql.metadata.Table tableHandle;
+  private final Iterable<Partition> partitions;
+
+  public TableSerializer(org.apache.hadoop.hive.ql.metadata.Table tableHandle,
+      Iterable<Partition> partitions) {
+    this.tableHandle = tableHandle;
+    this.partitions = partitions;
+  }
+
+  @Override
+  public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+      throws SemanticException, IOException {
+    if (cannotReplicateTable(additionalPropertiesProvider)) {
+      return;
+    }
+
+    Table tTable = tableHandle.getTTable();
+    tTable = addPropertiesToTable(tTable, additionalPropertiesProvider);
+    try {
+      TSerializer serializer = new TSerializer(new TJSONProtocol.Factory());
+      writer.jsonGenerator
+          .writeStringField(FIELD_NAME, serializer.toString(tTable, UTF_8));
+      writer.jsonGenerator.writeFieldName(PartitionSerializer.FIELD_NAME);
+      writePartitions(writer, additionalPropertiesProvider);
+    } catch (TException e) {
+      throw new SemanticException(ErrorMsg.ERROR_SERIALIZE_METASTORE.getMsg(), e);
+    }
+  }
+
+  private boolean cannotReplicateTable(ReplicationSpec additionalPropertiesProvider) {
+    return tableHandle == null || additionalPropertiesProvider.isNoop();
+  }
+
+  private Table addPropertiesToTable(Table table, ReplicationSpec additionalPropertiesProvider)
+      throws SemanticException, IOException {
+    if (additionalPropertiesProvider.isInReplicationScope()) {
+      table.putToParameters(
+            ReplicationSpec.KEY.CURR_STATE_ID.toString(),
+            additionalPropertiesProvider.getCurrentReplicationState());
+      if (isExternalTable(table)) {
+          // Replication destination will not be external - override if set
+        table.putToParameters("EXTERNAL", "FALSE");
+        }
+      if (isExternalTableType(table)) {
+          // Replication dest will not be external - override if set
+        table.setTableType(TableType.MANAGED_TABLE.toString());
+        }
+    } else {
+      // ReplicationSpec.KEY scopeKey = ReplicationSpec.KEY.REPL_SCOPE;
+      // write(out, ",\""+ scopeKey.toString() +"\":\"" + replicationSpec.get(scopeKey) +
"\"");
+      // TODO: if we want to be explicit about this dump not being a replication dump, we
can
+      // uncomment this else section, but currently unnneeded. Will require a lot of golden
file
+      // regen if we do so.
+    }
+    return table;
+  }
+
+  private boolean isExternalTableType(org.apache.hadoop.hive.metastore.api.Table table) {
+    return table.isSetTableType()
+        && table.getTableType().equalsIgnoreCase(TableType.EXTERNAL_TABLE.toString());
+  }
+
+  private boolean isExternalTable(org.apache.hadoop.hive.metastore.api.Table table) {
+    Map<String, String> params = table.getParameters();
+    return params.containsKey("EXTERNAL")
+        && params.get("EXTERNAL").equalsIgnoreCase("TRUE");
+  }
+
+  private void writePartitions(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+      throws SemanticException, IOException {
+    writer.jsonGenerator.writeStartArray();
+    if (partitions != null) {
+      for (org.apache.hadoop.hive.ql.metadata.Partition partition : partitions) {
+        new PartitionSerializer(partition.getTPartition())
+            .writeTo(writer, additionalPropertiesProvider);
+      }
+    }
+    writer.jsonGenerator.writeEndArray();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/VersionCompatibleSerializer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/VersionCompatibleSerializer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/VersionCompatibleSerializer.java
new file mode 100644
index 0000000..8201173
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/VersionCompatibleSerializer.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.io;
+
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.hive.ql.parse.EximUtil.METADATA_FORMAT_FORWARD_COMPATIBLE_VERSION;
+
+/**
+ * This is not used as of now as the conditional which lead to its usage is always false
+ * hence we have removed the conditional and the usage of this class, but might be required
in future.
+ */
+public class VersionCompatibleSerializer implements JsonWriter.Serializer {
+  @Override
+  public void writeTo(JsonWriter writer, ReplicationSpec additionalPropertiesProvider)
+      throws SemanticException, IOException {
+    writer.jsonGenerator.writeStringField("fcversion", METADATA_FORMAT_FORWARD_COMPATIBLE_VERSION);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java
index 9a4f8b9..1616ab9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AddPartitionHandler.java
@@ -35,7 +35,7 @@ import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.util.Iterator;
 
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 
 public class AddPartitionHandler extends AbstractHandler {
   protected AddPartitionHandler(NotificationEvent notificationEvent) {
@@ -108,7 +108,7 @@ public class AddPartitionHandler extends AbstractHandler {
   }
 
   @Override
-  public DUMPTYPE dumpType() {
-    return DUMPTYPE.EVENT_ADD_PARTITION;
+  public DumpType dumpType() {
+    return DumpType.EVENT_ADD_PARTITION;
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
index 20d04dc..b6c3496 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterPartitionHandler.java
@@ -23,14 +23,14 @@ import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.EximUtil;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
 
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
 
 public class AlterPartitionHandler extends AbstractHandler {
   private final org.apache.hadoop.hive.metastore.api.Partition after;
@@ -51,24 +51,24 @@ public class AlterPartitionHandler extends AbstractHandler {
   private enum Scenario {
     ALTER {
       @Override
-      DUMPTYPE dumpType() {
-        return DUMPTYPE.EVENT_ALTER_PARTITION;
+      DumpType dumpType() {
+        return DumpType.EVENT_ALTER_PARTITION;
       }
     },
     RENAME {
       @Override
-      DUMPTYPE dumpType() {
-        return DUMPTYPE.EVENT_RENAME_PARTITION;
+      DumpType dumpType() {
+        return DumpType.EVENT_RENAME_PARTITION;
       }
     },
     TRUNCATE {
       @Override
-      DUMPTYPE dumpType() {
-        return DUMPTYPE.EVENT_TRUNCATE_PARTITION;
+      DumpType dumpType() {
+        return DumpType.EVENT_TRUNCATE_PARTITION;
       }
     };
 
-    abstract DUMPTYPE dumpType();
+    abstract DumpType dumpType();
   }
 
   private Scenario scenarioType(org.apache.hadoop.hive.metastore.api.Partition before,
@@ -90,14 +90,14 @@ public class AlterPartitionHandler extends AbstractHandler {
     if (Scenario.ALTER == scenario) {
       withinContext.replicationSpec.setIsMetadataOnly(true);
       Table qlMdTable = new Table(tableObject);
-      List<Partition> qlPtns = new ArrayList<>();
-      qlPtns.add(new Partition(qlMdTable, after));
+      List<Partition> partitions = new ArrayList<>();
+      partitions.add(new Partition(qlMdTable, after));
       Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME);
       EximUtil.createExportDump(
           metaDataPath.getFileSystem(withinContext.hiveConf),
           metaDataPath,
           qlMdTable,
-          qlPtns,
+          partitions,
           withinContext.replicationSpec);
     }
     DumpMetaData dmd = withinContext.createDmd(this);
@@ -106,7 +106,7 @@ public class AlterPartitionHandler extends AbstractHandler {
   }
 
   @Override
-  public DUMPTYPE dumpType() {
+  public DumpType dumpType() {
     return scenario.dumpType();
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
index bfe0181..d553240 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/AlterTableHandler.java
@@ -22,13 +22,12 @@ import org.apache.hadoop.hive.metastore.api.NotificationEvent;
 import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.EximUtil;
-import org.apache.hadoop.hive.ql.parse.SemanticException;
 
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
 
 public class AlterTableHandler extends AbstractHandler {
-  private final org.apache.hadoop.hive.metastore.api.Table before;
   private final org.apache.hadoop.hive.metastore.api.Table after;
   private final boolean isTruncateOp;
   private final Scenario scenario;
@@ -36,30 +35,30 @@ public class AlterTableHandler extends AbstractHandler {
   private enum Scenario {
     ALTER {
       @Override
-      DUMPTYPE dumpType() {
-        return DUMPTYPE.EVENT_ALTER_TABLE;
+      DumpType dumpType() {
+        return DumpType.EVENT_ALTER_TABLE;
       }
     },
     RENAME {
       @Override
-      DUMPTYPE dumpType() {
-        return DUMPTYPE.EVENT_RENAME_TABLE;
+      DumpType dumpType() {
+        return DumpType.EVENT_RENAME_TABLE;
       }
     },
     TRUNCATE {
       @Override
-      DUMPTYPE dumpType() {
-        return DUMPTYPE.EVENT_TRUNCATE_TABLE;
+      DumpType dumpType() {
+        return DumpType.EVENT_TRUNCATE_TABLE;
       }
     };
 
-    abstract DUMPTYPE dumpType();
+    abstract DumpType dumpType();
   }
 
   AlterTableHandler(NotificationEvent event) throws Exception {
     super(event);
     AlterTableMessage atm = deserializer.getAlterTableMessage(event.getMessage());
-    before = atm.getTableObjBefore();
+    org.apache.hadoop.hive.metastore.api.Table before = atm.getTableObjBefore();
     after = atm.getTableObjAfter();
     isTruncateOp = atm.getIsTruncateOp();
     scenario = scenarioType(before, after);
@@ -97,7 +96,7 @@ public class AlterTableHandler extends AbstractHandler {
   }
 
   @Override
-  public DUMPTYPE dumpType() {
+  public DumpType dumpType() {
     return scenario.dumpType();
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java
index 03f400d..88600fd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/CreateTableHandler.java
@@ -28,7 +28,7 @@ import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 
 public class CreateTableHandler extends AbstractHandler {
 
@@ -80,7 +80,7 @@ public class CreateTableHandler extends AbstractHandler {
   }
 
   @Override
-  public DUMPTYPE dumpType() {
-    return DUMPTYPE.EVENT_CREATE_TABLE;
+  public DumpType dumpType() {
+    return DumpType.EVENT_CREATE_TABLE;
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java
index 61c5f37..78cd74f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DefaultHandler.java
@@ -19,8 +19,9 @@ package org.apache.hadoop.hive.ql.parse.repl.events;
 
 import org.apache.hadoop.hive.metastore.api.NotificationEvent;
 
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
 
 public class DefaultHandler extends AbstractHandler {
 
@@ -37,7 +38,7 @@ public class DefaultHandler extends AbstractHandler {
   }
 
   @Override
-  public DUMPTYPE dumpType() {
-    return DUMPTYPE.EVENT_UNKNOWN;
+  public DumpType dumpType() {
+    return DumpType.EVENT_UNKNOWN;
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java
index 3ad794e..c4a0908 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropPartitionHandler.java
@@ -19,8 +19,9 @@ package org.apache.hadoop.hive.ql.parse.repl.events;
 
 import org.apache.hadoop.hive.metastore.api.NotificationEvent;
 
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
 
 public class DropPartitionHandler extends AbstractHandler {
 
@@ -37,7 +38,7 @@ public class DropPartitionHandler extends AbstractHandler {
   }
 
   @Override
-  public DUMPTYPE dumpType() {
-    return DUMPTYPE.EVENT_DROP_PARTITION;
+  public DumpType dumpType() {
+    return DumpType.EVENT_DROP_PARTITION;
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java
index cae379b..e3addaf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/DropTableHandler.java
@@ -19,8 +19,9 @@ package org.apache.hadoop.hive.ql.parse.repl.events;
 
 import org.apache.hadoop.hive.metastore.api.NotificationEvent;
 
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
 
 public class DropTableHandler extends AbstractHandler {
 
@@ -37,7 +38,7 @@ public class DropTableHandler extends AbstractHandler {
   }
 
   @Override
-  public DUMPTYPE dumpType() {
-    return DUMPTYPE.EVENT_DROP_TABLE;
+  public DumpType dumpType() {
+    return DumpType.EVENT_DROP_TABLE;
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java
index 199145a..29f3b42 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/EventHandler.java
@@ -22,8 +22,8 @@ import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
 
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 
 public interface EventHandler {
   void handle(Context withinContext) throws Exception;
@@ -32,7 +32,7 @@ public interface EventHandler {
 
   long toEventId();
 
-  DUMPTYPE dumpType();
+  DumpType dumpType();
 
   class Context {
     final Path eventRoot, cmRoot;

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java
index e9f2a6a..910b396 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/events/InsertHandler.java
@@ -32,8 +32,9 @@ import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DUMPTYPE;
-import static org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer.DumpMetaData;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;
 
 public class InsertHandler extends AbstractHandler {
 
@@ -103,7 +104,7 @@ public class InsertHandler extends AbstractHandler {
   }
 
   @Override
-  public DUMPTYPE dumpType() {
-    return DUMPTYPE.EVENT_INSERT;
+  public DumpType dumpType() {
+    return DumpType.EVENT_INSERT;
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
new file mode 100644
index 0000000..12ad19b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/DumpMetaData.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.ReplChangeManager;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.dump.Utils;
+
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+
+public class DumpMetaData {
+  // wrapper class for reading and writing metadata about a dump
+  // responsible for _dumpmetadata files
+  private static final String DUMP_METADATA = "_dumpmetadata";
+
+  private DumpType dumpType;
+  private Long eventFrom = null;
+  private Long eventTo = null;
+  private String payload = null;
+  private boolean initialized = false;
+
+  private final Path dumpFile;
+  private final HiveConf hiveConf;
+  private Path cmRoot;
+
+  public DumpMetaData(Path dumpRoot, HiveConf hiveConf) {
+    this.hiveConf = hiveConf;
+    dumpFile = new Path(dumpRoot, DUMP_METADATA);
+  }
+
+  public DumpMetaData(Path dumpRoot, DumpType lvl, Long eventFrom, Long eventTo, Path cmRoot,
+      HiveConf hiveConf) {
+    this(dumpRoot, hiveConf);
+    setDump(lvl, eventFrom, eventTo, cmRoot);
+  }
+
+  public void setDump(DumpType lvl, Long eventFrom, Long eventTo, Path cmRoot) {
+    this.dumpType = lvl;
+    this.eventFrom = eventFrom;
+    this.eventTo = eventTo;
+    this.initialized = true;
+    this.cmRoot = cmRoot;
+  }
+
+  private void loadDumpFromFile() throws SemanticException {
+    try {
+      // read from dumpfile and instantiate self
+      FileSystem fs = dumpFile.getFileSystem(hiveConf);
+      BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(dumpFile)));
+      String line = null;
+      if ((line = br.readLine()) != null) {
+        String[] lineContents = line.split("\t", 5);
+        setDump(DumpType.valueOf(lineContents[0]), Long.valueOf(lineContents[1]),
+            Long.valueOf(lineContents[2]),
+            new Path(lineContents[3]));
+        setPayload(lineContents[4].equals(Utilities.nullStringOutput) ? null : lineContents[4]);
+        ReplChangeManager.setCmRoot(cmRoot);
+      } else {
+        throw new IOException(
+            "Unable to read valid values from dumpFile:" + dumpFile.toUri().toString());
+      }
+    } catch (IOException ioe) {
+      throw new SemanticException(ioe);
+    }
+  }
+
+  public DumpType getDumpType() throws SemanticException {
+    initializeIfNot();
+    return this.dumpType;
+  }
+
+  public String getPayload() throws SemanticException {
+    initializeIfNot();
+    return this.payload;
+  }
+
+  public void setPayload(String payload) {
+    this.payload = payload;
+  }
+
+  public Long getEventFrom() throws SemanticException {
+    initializeIfNot();
+    return eventFrom;
+  }
+
+  public Long getEventTo() throws SemanticException {
+    initializeIfNot();
+    return eventTo;
+  }
+
+  public Path getDumpFilePath() {
+    return dumpFile;
+  }
+
+  public boolean isIncrementalDump() throws SemanticException {
+    initializeIfNot();
+    return (this.dumpType == DumpType.INCREMENTAL);
+  }
+
+  private void initializeIfNot() throws SemanticException {
+    if (!initialized) {
+      loadDumpFromFile();
+    }
+  }
+
+
+  public void write() throws SemanticException {
+    Utils.writeOutput(
+        Arrays.asList(
+            dumpType.toString(),
+            eventFrom.toString(),
+            eventTo.toString(),
+            cmRoot.toString(),
+            payload),
+        dumpFile,
+        hiveConf
+    );
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
new file mode 100644
index 0000000..fc02dfd
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetaData.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+
+/**
+ * Utility class to help return complex value from readMetaData function
+ */
+public class MetaData {
+  private final Database db;
+  private final Table table;
+  private final Iterable<Partition> partitions;
+  private final ReplicationSpec replicationSpec;
+  public final Function function;
+
+  public MetaData() {
+    this(null, null, null, new ReplicationSpec(), null);
+  }
+
+  MetaData(Database db, Table table, Iterable<Partition> partitions,
+      ReplicationSpec replicationSpec, Function function) {
+    this.db = db;
+    this.table = table;
+    this.partitions = partitions;
+    this.replicationSpec = replicationSpec;
+    this.function = function;
+  }
+
+  public Database getDatabase() {
+    return db;
+  }
+
+  public Table getTable() {
+    return table;
+  }
+
+  public Iterable<Partition> getPartitions() {
+    return partitions;
+  }
+
+  public ReplicationSpec getReplicationSpec() {
+    return replicationSpec;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetadataJson.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetadataJson.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetadataJson.java
new file mode 100644
index 0000000..b7a5680
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/MetadataJson.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Function;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.repl.dump.io.DBSerializer;
+import org.apache.hadoop.hive.ql.parse.repl.dump.io.FunctionSerializer;
+import org.apache.hadoop.hive.ql.parse.repl.dump.io.PartitionSerializer;
+import org.apache.hadoop.hive.ql.parse.repl.dump.io.TableSerializer;
+import org.apache.thrift.TBase;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TException;
+import org.apache.thrift.protocol.TJSONProtocol;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import javax.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.hive.ql.parse.repl.dump.io.JsonWriter.Serializer.UTF_8;
+
+public class MetadataJson {
+  private final JSONObject json;
+  private final TDeserializer deserializer;
+  private final String tableDesc;
+
+  public MetadataJson(String message) throws JSONException, SemanticException {
+    deserializer = new TDeserializer(new TJSONProtocol.Factory());
+    json = new JSONObject(message);
+    checkCompatibility();
+    tableDesc = jsonEntry(TableSerializer.FIELD_NAME);
+  }
+
+  public MetaData getMetaData() throws TException, JSONException {
+    return new MetaData(
+        database(),
+        table(),
+        partitions(),
+        readReplicationSpec(),
+        function()
+    );
+  }
+
+  private Function function() throws TException {
+    return deserialize(new Function(), jsonEntry(FunctionSerializer.FIELD_NAME));
+  }
+
+  private Database database() throws TException {
+    return deserialize(new Database(), jsonEntry(DBSerializer.FIELD_NAME));
+  }
+
+  private Table table() throws TException {
+    return deserialize(new Table(), tableDesc);
+  }
+
+  private <T extends TBase> T deserialize(T intoObject, String json) throws TException
{
+    if (json == null) {
+      return null;
+    }
+    deserializer.deserialize(intoObject, json, UTF_8);
+    return intoObject;
+  }
+
+  private List<Partition> partitions() throws JSONException, TException {
+    if (tableDesc == null) {
+      return null;
+    }
+    // TODO : jackson-streaming-iterable-redo this
+    JSONArray jsonPartitions = new JSONArray(json.getString(PartitionSerializer.FIELD_NAME));
+    List<Partition> partitionsList = new ArrayList<>(jsonPartitions.length());
+    for (int i = 0; i < jsonPartitions.length(); ++i) {
+      String partDesc = jsonPartitions.getString(i);
+      partitionsList.add(deserialize(new Partition(), partDesc));
+    }
+    return partitionsList;
+  }
+
+  private ReplicationSpec readReplicationSpec() {
+    com.google.common.base.Function<String, String> keyFetcher =
+        new com.google.common.base.Function<String, String>() {
+          @Override
+          public String apply(@Nullable String s) {
+            return jsonEntry(s);
+          }
+        };
+    return new ReplicationSpec(keyFetcher);
+  }
+
+  private void checkCompatibility() throws SemanticException, JSONException {
+    String version = json.getString("version");
+    String fcVersion = jsonEntry("fcversion");
+    EximUtil.doCheckCompatibility(
+        EximUtil.METADATA_FORMAT_VERSION,
+        version,
+        fcVersion);
+  }
+
+  private String jsonEntry(String forName) {
+    try {
+      return json.getString(forName);
+    } catch (JSONException ignored) {
+      return null;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapperTest.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapperTest.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapperTest.java
new file mode 100644
index 0000000..3028e76
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/dump/HiveWrapperTest.java
@@ -0,0 +1,27 @@
+package org.apache.hadoop.hive.ql.parse.repl.dump;
+
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InOrder;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
+public class HiveWrapperTest {
+  @Mock
+  private HiveWrapper.Tuple.Function<ReplicationSpec> specFunction;
+  @Mock
+  private HiveWrapper.Tuple.Function<Table> tableFunction;
+
+  @Test
+  public void replicationIdIsRequestedBeforeObjectDefinition() throws HiveException {
+    new HiveWrapper.Tuple<>(specFunction, tableFunction);
+    InOrder inOrder = Mockito.inOrder(specFunction, tableFunction);
+    inOrder.verify(specFunction).fromMetaStore();
+    inOrder.verify(tableFunction).fromMetaStore();
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/9e9356b5/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java
b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java
index d44cb79..4b802c4 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/repl/events/TestEventHandlerFactory.java
@@ -19,7 +19,7 @@
 package org.apache.hadoop.hive.ql.parse.repl.events;
 
 import org.apache.hadoop.hive.metastore.api.NotificationEvent;
-import org.apache.hadoop.hive.ql.parse.ReplicationSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
 import org.junit.Test;
 
 import static org.junit.Assert.assertTrue;
@@ -44,7 +44,7 @@ public class TestEventHandlerFactory {
       }
 
       @Override
-      public ReplicationSemanticAnalyzer.DUMPTYPE dumpType() {
+      public DumpType dumpType() {
         return null;
       }
     }


Mime
View raw message