hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject [5/6] hive git commit: HIVE-17100 : Improve HS2 operation logs for REPL commands (Sankar Hariappan, reviewed by Anishek Agarwal, Thejas Nair)
Date Thu, 31 Aug 2017 00:22:24 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java
new file mode 100644
index 0000000..a003d37
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/ReplState.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class ReplState {
+  @JsonIgnoreProperties
+  private static final Logger REPL_LOG = LoggerFactory.getLogger("ReplState");
+
+  @JsonIgnoreProperties
+  private static final ObjectMapper mapper = new ObjectMapper(); // Thread-safe.
+
+  static {
+    mapper.configure(SerializationConfig.Feature.AUTO_DETECT_GETTERS, false);
+    mapper.configure(SerializationConfig.Feature.AUTO_DETECT_IS_GETTERS, false);
+    mapper.configure(SerializationConfig.Feature.AUTO_DETECT_FIELDS, false);
+  }
+
+  public enum LogTag {
+    START,
+    TABLE_DUMP,
+    FUNCTION_DUMP,
+    EVENT_DUMP,
+    TABLE_LOAD,
+    FUNCTION_LOAD,
+    EVENT_LOAD,
+    END
+  }
+
+  public void log(LogTag tag) {
+    try {
+      REPL_LOG.info("REPL::{}: {}", tag.name(), mapper.writeValueAsString(this));
+    } catch (Exception exception) {
+      REPL_LOG.error("Could not serialize REPL log: {}", exception.getMessage());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
index f40c703..a48a17e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
@@ -31,6 +31,7 @@ import com.google.common.collect.Collections2;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.List;
 
 public class Utils {
@@ -64,14 +65,18 @@ public class Utils {
   public static Iterable<? extends String> matchesTbl(Hive db, String dbName, String tblPattern)
       throws HiveException {
     if (tblPattern == null) {
-      return Collections2.filter(db.getAllTables(dbName),
-          tableName -> {
-            assert tableName != null;
-            return !tableName.toLowerCase().startsWith(
-                SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase());
-          });
+      return getAllTables(db, dbName);
     } else {
       return db.getTablesByPattern(dbName, tblPattern);
     }
   }
+
+  public static Collection<String> getAllTables(Hive db, String dbName) throws HiveException {
+    return Collections2.filter(db.getAllTables(dbName),
+            tableName -> {
+              assert tableName != null;
+              return !tableName.toLowerCase().startsWith(
+                      SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase());
+            });
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
new file mode 100644
index 0000000..1b0ec59
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/BootstrapDumpLogger.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.BootstrapDumpBegin;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.BootstrapDumpEnd;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.BootstrapDumpFunction;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.BootstrapDumpTable;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState.LogTag;
+
+public class BootstrapDumpLogger extends ReplLogger {
+  private String dbName;
+  private String dumpDir;
+  private long estimatedNumTables;
+  private long estimatedNumFunctions;
+  private long tableSeqNo;
+  private long functionSeqNo;
+
+  public BootstrapDumpLogger(String dbName, String dumpDir,
+                             int estimatedNumTables, int estimatedNumFunctions) {
+    this.dbName = dbName;
+    this.dumpDir = dumpDir;
+    this.estimatedNumTables = estimatedNumTables;
+    this.estimatedNumFunctions = estimatedNumFunctions;
+    this.tableSeqNo = 0;
+    this.functionSeqNo = 0;
+  }
+
+  @Override
+  public void startLog() {
+    (new BootstrapDumpBegin(dbName, estimatedNumTables, estimatedNumFunctions))
+            .log(LogTag.START);
+  }
+
+  @Override
+  public void tableLog(String tableName, TableType tableType) {
+    tableSeqNo++;
+    (new BootstrapDumpTable(dbName, tableName, tableType, tableSeqNo, estimatedNumTables))
+            .log(LogTag.TABLE_DUMP);
+  }
+
+  @Override
+  public void functionLog(String funcName) {
+    functionSeqNo++;
+    (new BootstrapDumpFunction(dbName, funcName, functionSeqNo, estimatedNumFunctions))
+            .log(LogTag.FUNCTION_DUMP);
+  }
+
+  @Override
+  public void endLog(String lastReplId) {
+    (new BootstrapDumpEnd(dbName, tableSeqNo, functionSeqNo, dumpDir, lastReplId))
+            .log(LogTag.END);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/IncrementalDumpLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/IncrementalDumpLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/IncrementalDumpLogger.java
new file mode 100644
index 0000000..03144df
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/IncrementalDumpLogger.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log;
+
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.IncrementalDumpBegin;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.IncrementalDumpEnd;
+import org.apache.hadoop.hive.ql.parse.repl.dump.log.state.IncrementalDumpEvent;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState.LogTag;
+
+public class IncrementalDumpLogger extends ReplLogger {
+  private String dbName;
+  private String dumpDir;
+  private long estimatedNumEvents;
+  private long eventSeqNo;
+
+  public IncrementalDumpLogger(String dbName, String dumpDir, long estimatedNumEvents) {
+    this.dbName = dbName;
+    this.dumpDir = dumpDir;
+    this.estimatedNumEvents = estimatedNumEvents;
+    this.eventSeqNo = 0;
+  }
+
+  @Override
+  public void startLog() {
+    (new IncrementalDumpBegin(dbName, estimatedNumEvents)).log(LogTag.START);
+  }
+
+  @Override
+  public void eventLog(String eventId, String eventType) {
+    eventSeqNo++;
+    (new IncrementalDumpEvent(dbName, eventId, eventType, eventSeqNo, estimatedNumEvents))
+            .log(LogTag.EVENT_DUMP);
+  }
+
+  @Override
+  public void endLog(String lastReplId) {
+    (new IncrementalDumpEnd(dbName, eventSeqNo, dumpDir, lastReplId)).log(LogTag.END);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java
new file mode 100644
index 0000000..3c4ebd3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpBegin.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapDumpBegin extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private DumpType dumpType;
+
+  @JsonProperty
+  private Long estimatedNumTables;
+
+  @JsonProperty
+  private Long estimatedNumFunctions;
+
+  @JsonProperty
+  private Long dumpStartTime;
+
+  public BootstrapDumpBegin(String dbName, long estimatedNumTables, long estimatedNumFunctions) {
+    this.dbName = dbName;
+    this.dumpType = DumpType.BOOTSTRAP;
+    this.estimatedNumTables = estimatedNumTables;
+    this.estimatedNumFunctions = estimatedNumFunctions;
+    this.dumpStartTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java
new file mode 100644
index 0000000..a4dcfb5
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpEnd.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapDumpEnd extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private DumpType dumpType;
+
+  @JsonProperty
+  private Long actualNumTables;
+
+  @JsonProperty
+  private Long actualNumFunctions;
+
+  @JsonProperty
+  private Long dumpEndTime;
+
+  @JsonProperty
+  private String dumpDir;
+
+  @JsonProperty
+  private String lastReplId;
+
+  public BootstrapDumpEnd(String dbName,
+                          long actualNumTables,
+                          long actualNumFunctions,
+                          String dumpDir,
+                          String lastReplId) {
+    this.dbName = dbName;
+    this.dumpType = DumpType.BOOTSTRAP;
+    this.actualNumTables = actualNumTables;
+    this.actualNumFunctions = actualNumFunctions;
+    this.dumpEndTime = System.currentTimeMillis() / 1000;
+    this.dumpDir = dumpDir;
+    this.lastReplId = lastReplId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java
new file mode 100644
index 0000000..a414daa
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpFunction.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapDumpFunction extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private String functionName;
+
+  @JsonProperty
+  private String functionsDumpProgress;
+
+  @JsonProperty
+  private Long dumpTime;
+
+  public BootstrapDumpFunction(String dbName, String funcName,
+                               long functionSeqNo, long estimatedNumFunctions) {
+    this.dbName = dbName;
+    this.functionName = funcName;
+    this.functionsDumpProgress = new String(new StringBuilder()
+                                  .append(functionSeqNo).append("/").append(estimatedNumFunctions));
+    this.dumpTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java
new file mode 100644
index 0000000..4bf4905
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/BootstrapDumpTable.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapDumpTable extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private String tableName;
+
+  @JsonProperty
+  private String tableType;
+
+  @JsonProperty
+  private String tablesDumpProgress;
+
+  @JsonProperty
+  private Long dumpTime;
+
+  public BootstrapDumpTable(String dbName,
+                            String tableName,
+                            TableType tableType,
+                            long tableSeqNo,
+                            long estimatedNumTables) {
+    this.dbName = dbName;
+    this.tableName = tableName;
+    this.tableType = tableType.name();
+    this.tablesDumpProgress = new String(new StringBuilder()
+                                        .append(tableSeqNo).append("/").append(estimatedNumTables));
+    this.dumpTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java
new file mode 100644
index 0000000..166d359
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpBegin.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalDumpBegin extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private DumpType dumpType;
+
+  @JsonProperty
+  private Long estimatedNumEvents;
+
+  @JsonProperty
+  private Long dumpStartTime;
+
+  public IncrementalDumpBegin(String dbName, long estimatedNumEvents) {
+    this.dbName = dbName;
+    this.dumpType = DumpType.INCREMENTAL;
+    this.estimatedNumEvents = estimatedNumEvents;
+    this.dumpStartTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java
new file mode 100644
index 0000000..a7cec21
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEnd.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalDumpEnd extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private DumpType dumpType;
+
+  @JsonProperty
+  private Long actualNumEvents;
+
+  @JsonProperty
+  private Long dumpEndTime;
+
+  @JsonProperty
+  private String dumpDir;
+
+  @JsonProperty
+  private String lastReplId;
+
+  public IncrementalDumpEnd(String dbName,
+                            long actualNumEvents,
+                            String dumpDir,
+                            String lastReplId) {
+    this.dbName = dbName;
+    this.dumpType = DumpType.INCREMENTAL;
+    this.actualNumEvents = actualNumEvents;
+    this.dumpEndTime = System.currentTimeMillis() / 1000;
+    this.dumpDir = dumpDir;
+    this.lastReplId = lastReplId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java
new file mode 100644
index 0000000..ebeb468
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/log/state/IncrementalDumpEvent.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.dump.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalDumpEvent extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private String eventId;
+
+  @JsonProperty
+  private String eventType;
+
+  @JsonProperty
+  private String eventsDumpProgress;
+
+  @JsonProperty
+  private Long dumpTime;
+
+  public IncrementalDumpEvent(String dbName,
+                              String eventId,
+                              String eventType,
+                              long eventSeqNo,
+                              long estimatedNumEvents) {
+    this.dbName = dbName;
+    this.eventId = eventId;
+    this.eventType = eventType;
+    this.eventsDumpProgress = new String(new StringBuilder()
+                                        .append(eventSeqNo).append("/").append(estimatedNumEvents));
+    this.dumpTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/BootstrapLoadLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/BootstrapLoadLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/BootstrapLoadLogger.java
new file mode 100644
index 0000000..739bd12
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/BootstrapLoadLogger.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.parse.repl.load.log.state.*;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState.LogTag;
+
+public class BootstrapLoadLogger extends ReplLogger {
+  private String dbName;
+  private String dumpDir;
+  private long numTables;
+  private long numFunctions;
+  private long tableSeqNo;
+  private long functionSeqNo;
+
+  public BootstrapLoadLogger(String dbName, String dumpDir, long numTables, long numFunctions) {
+    this.dbName = dbName;
+    this.dumpDir = dumpDir;
+    this.numTables = numTables;
+    this.numFunctions = numFunctions;
+    this.tableSeqNo = 0;
+    this.functionSeqNo = 0;
+  }
+
+  @Override
+  public void startLog() {
+    (new BootstrapLoadBegin(dbName, dumpDir, numTables, numFunctions)).log(LogTag.START);
+  }
+
+  @Override
+  public void tableLog(String tableName, TableType tableType) {
+    tableSeqNo++;
+    (new BootstrapLoadTable(dbName, tableName, tableType, tableSeqNo, numTables))
+            .log(LogTag.TABLE_LOAD);
+  }
+
+  @Override
+  public void functionLog(String funcName) {
+    functionSeqNo++;
+    (new BootstrapLoadFunction(dbName, funcName, functionSeqNo, numFunctions))
+            .log(LogTag.FUNCTION_LOAD);
+  }
+
+  @Override
+  public void endLog(String lastReplId) {
+    (new BootstrapLoadEnd(dbName, numTables, numFunctions, dumpDir, lastReplId))
+            .log(LogTag.END);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/IncrementalLoadLogger.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/IncrementalLoadLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/IncrementalLoadLogger.java
new file mode 100644
index 0000000..d2114f0
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/IncrementalLoadLogger.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log;
+
+import org.apache.hadoop.hive.ql.parse.repl.load.log.state.IncrementalLoadBegin;
+import org.apache.hadoop.hive.ql.parse.repl.load.log.state.IncrementalLoadEnd;
+import org.apache.hadoop.hive.ql.parse.repl.load.log.state.IncrementalLoadEvent;
+import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState.LogTag;
+
+public class IncrementalLoadLogger extends ReplLogger {
+  private String dbName;
+  private String dumpDir;
+  private long numEvents;
+  private long eventSeqNo;
+
+  public IncrementalLoadLogger(String dbName, String dumpDir, int numEvents) {
+    this.dbName = dbName;
+    this.dumpDir = dumpDir;
+    this.numEvents = numEvents;
+    this.eventSeqNo = 0;
+  }
+
+  @Override
+  public void startLog() {
+    (new IncrementalLoadBegin(dbName, dumpDir, numEvents)).log(LogTag.START);
+  }
+
+  @Override
+  public void eventLog(String eventId, String eventType) {
+    eventSeqNo++;
+    (new IncrementalLoadEvent(dbName, eventId, eventType, eventSeqNo, numEvents))
+            .log(LogTag.EVENT_LOAD);
+  }
+
+  @Override
+  public void endLog(String lastReplId) {
+    (new IncrementalLoadEnd(dbName, numEvents, dumpDir, lastReplId)).log(LogTag.END);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java
new file mode 100644
index 0000000..5574454
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadBegin.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapLoadBegin extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private String dumpDir;
+
+  @JsonProperty
+  private DumpType loadType;
+
+  @JsonProperty
+  private Long numTables;
+
+  @JsonProperty
+  private Long numFunctions;
+
+  @JsonProperty
+  private Long loadStartTime;
+
+  public BootstrapLoadBegin(String dbName, String dumpDir, long numTables, long numFunctions) {
+    this.dbName = dbName;
+    this.dumpDir = dumpDir;
+    this.loadType = DumpType.BOOTSTRAP;
+    this.numTables = numTables;
+    this.numFunctions = numFunctions;
+    this.loadStartTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java
new file mode 100644
index 0000000..3db8144
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadEnd.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapLoadEnd extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private DumpType loadType;
+
+  @JsonProperty
+  private Long numTables;
+
+  @JsonProperty
+  private Long numFunctions;
+
+  @JsonProperty
+  private Long loadEndTime;
+
+  @JsonProperty
+  private String dumpDir;
+
+  @JsonProperty
+  private String lastReplId;
+
+  public BootstrapLoadEnd(String dbName,
+                          long numTables,
+                          long numFunctions,
+                          String dumpDir,
+                          String lastReplId) {
+    this.dbName = dbName;
+    this.loadType = DumpType.BOOTSTRAP;
+    this.numTables = numTables;
+    this.numFunctions = numFunctions;
+    this.loadEndTime = System.currentTimeMillis() / 1000;
+    this.dumpDir = dumpDir;
+    this.lastReplId = lastReplId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java
new file mode 100644
index 0000000..e2434ce
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadFunction.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapLoadFunction extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private String functionName;
+
+  @JsonProperty
+  private String functionsLoadProgress;
+
+  @JsonProperty
+  private Long loadTime;
+
+  public BootstrapLoadFunction(String dbName, String funcName,
+                               long functionSeqNo, long numFunctions) {
+    this.dbName = dbName;
+    this.functionName = funcName;
+    this.functionsLoadProgress = new String(new StringBuilder()
+                                          .append(functionSeqNo).append("/").append(numFunctions));
+    this.loadTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadTable.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadTable.java
new file mode 100644
index 0000000..63ff467
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/BootstrapLoadTable.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class BootstrapLoadTable extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private String tableName;
+
+  @JsonProperty
+  private String tableType;
+
+  @JsonProperty
+  private String tablesLoadProgress;
+
+  @JsonProperty
+  private Long loadTime;
+
+  public BootstrapLoadTable(String dbName,
+                            String tableName,
+                            TableType tableType,
+                            long tableSeqNo,
+                            long numTables) {
+    this.dbName = dbName;
+    this.tableName = tableName;
+    this.tableType = tableType.name();
+    this.tablesLoadProgress = new String(new StringBuilder()
+                                          .append(tableSeqNo).append("/").append(numTables));
+    this.loadTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadBegin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadBegin.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadBegin.java
new file mode 100644
index 0000000..93f1d7b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadBegin.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalLoadBegin extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private String dumpDir;
+
+  @JsonProperty
+  private DumpType loadType;
+
+  @JsonProperty
+  private Long numEvents;
+
+  @JsonProperty
+  private Long loadStartTime;
+
+  public IncrementalLoadBegin(String dbName, String dumpDir, long numEvents) {
+    this.dbName = dbName;
+    this.dumpDir = dumpDir;
+    this.loadType = DumpType.INCREMENTAL;
+    this.numEvents = numEvents;
+    this.loadStartTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEnd.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEnd.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEnd.java
new file mode 100644
index 0000000..f633c72
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEnd.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.apache.hadoop.hive.ql.parse.repl.DumpType;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalLoadEnd extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private DumpType loadType;
+
+  @JsonProperty
+  private Long numEvents;
+
+  @JsonProperty
+  private Long loadEndTime;
+
+  @JsonProperty
+  private String dumpDir;
+
+  @JsonProperty
+  private String lastReplId;
+
+  public IncrementalLoadEnd(String dbName,
+                            long numEvents,
+                            String dumpDir,
+                            String lastReplId) {
+    this.dbName = dbName;
+    this.loadType = DumpType.INCREMENTAL;
+    this.numEvents = numEvents;
+    this.loadEndTime = System.currentTimeMillis() / 1000;
+    this.dumpDir = dumpDir;
+    this.lastReplId = lastReplId;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEvent.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEvent.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEvent.java
new file mode 100644
index 0000000..224aeb2
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/log/state/IncrementalLoadEvent.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse.repl.load.log.state;
+
+import org.apache.hadoop.hive.ql.parse.repl.ReplState;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class IncrementalLoadEvent extends ReplState {
+  @JsonProperty
+  private String dbName;
+
+  @JsonProperty
+  private String eventId;
+
+  @JsonProperty
+  private String eventType;
+
+  @JsonProperty
+  private String eventsLoadProgress;
+
+  @JsonProperty
+  private Long loadTime;
+
+  public IncrementalLoadEvent(String dbName,
+                              String eventId,
+                              String eventType,
+                              long eventSeqNo,
+                              long numEvents) {
+    this.dbName = dbName;
+    this.eventId = eventId;
+    this.eventType = eventType;
+    this.eventsLoadProgress = new String(new StringBuilder()
+                                            .append(eventSeqNo).append("/").append(numEvents));
+    this.loadTime = System.currentTimeMillis() / 1000;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
index 3f176aa..caf6f3f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
@@ -49,12 +49,20 @@ import java.util.List;
 import static org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.toReadEntity;
 
 public class CreateFunctionHandler extends AbstractMessageHandler {
+  private String functionName;
+
+  public String getFunctionName() {
+    return functionName;
+  }
+
   @Override
   public List<Task<? extends Serializable>> handle(Context context)
       throws SemanticException {
     try {
       FunctionDescBuilder builder = new FunctionDescBuilder(context);
       CreateFunctionDesc descToLoad = builder.build();
+      this.functionName = builder.metadata.function.getFunctionName();
+
       context.log.debug("Loading function desc : {}", descToLoad.toString());
       Task<FunctionWork> createTask = TaskFactory.get(
           new FunctionWork(descToLoad), context.hiveConf

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java
index 8daff6d..3ccd639 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/MessageHandler.java
@@ -45,9 +45,10 @@ public interface MessageHandler {
   UpdatedMetaDataTracker getUpdatedMetadata();
 
   class Context {
-    final String dbName, tableName, location;
-    final Task<? extends Serializable> precursor;
-    DumpMetaData dmd;
+    public String dbName;
+    public final String tableName, location;
+    public final Task<? extends Serializable> precursor;
+    public DumpMetaData dmd;
     final HiveConf hiveConf;
     final Hive db;
     final org.apache.hadoop.hive.ql.Context nestedContext;
@@ -83,7 +84,7 @@ public interface MessageHandler {
       return StringUtils.isEmpty(tableName);
     }
 
-    boolean isDbNameEmpty() {
+    public boolean isDbNameEmpty() {
       return StringUtils.isEmpty(dbName);
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/70cb7f0b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
index a4fb378..1770046 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.Order;
 import org.apache.hadoop.hive.ql.exec.Task;
@@ -34,11 +35,8 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Table;
 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
-import org.apache.hadoop.hive.ql.parse.EximUtil;
 import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
-import org.apache.hadoop.hive.ql.plan.CreateViewDesc;
 
 /**
  * ImportTableDesc.
@@ -56,7 +54,7 @@ public class ImportTableDesc {
     this.dbName = dbName;
     this.table = table;
 
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         this.createTblDesc = new CreateTableDesc(dbName,
                 table.getTableName(),
@@ -122,7 +120,7 @@ public class ImportTableDesc {
     }
   }
 
-  public TYPE getTableType() {
+  public TYPE getDescType() {
     if (table.isView() || table.isMaterializedView()) {
       return TYPE.VIEW;
     }
@@ -143,7 +141,7 @@ public class ImportTableDesc {
   }
 
   public void setReplicationSpec(ReplicationSpec replSpec) {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         createTblDesc.setReplicationSpec(replSpec);
         break;
@@ -154,20 +152,20 @@ public class ImportTableDesc {
   }
 
   public void setExternal(boolean isExternal) {
-    if (TYPE.TABLE.equals(getTableType())) {
+    if (TYPE.TABLE.equals(getDescType())) {
       createTblDesc.setExternal(isExternal);
     }
   }
 
   public boolean isExternal() {
-    if (TYPE.TABLE.equals(getTableType())) {
+    if (TYPE.TABLE.equals(getDescType())) {
       return createTblDesc.isExternal();
     }
     return false;
   }
 
   public void setLocation(String location) {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         createTblDesc.setLocation(location);
         break;
@@ -178,7 +176,7 @@ public class ImportTableDesc {
   }
 
   public String getLocation() {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return createTblDesc.getLocation();
       case VIEW:
@@ -188,7 +186,7 @@ public class ImportTableDesc {
   }
 
   public void setTableName(String tableName) throws SemanticException {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         createTblDesc.setTableName(tableName);
         break;
@@ -201,7 +199,7 @@ public class ImportTableDesc {
   }
 
   public String getTableName() throws SemanticException {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return createTblDesc.getTableName();
       case VIEW:
@@ -213,7 +211,7 @@ public class ImportTableDesc {
   }
 
   public List<FieldSchema> getPartCols() {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return createTblDesc.getPartCols();
       case VIEW:
@@ -223,7 +221,7 @@ public class ImportTableDesc {
   }
 
   public List<FieldSchema> getCols() {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return createTblDesc.getCols();
       case VIEW:
@@ -233,7 +231,7 @@ public class ImportTableDesc {
   }
 
   public Map<String, String> getTblProps() {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return createTblDesc.getTblProps();
       case VIEW:
@@ -243,7 +241,7 @@ public class ImportTableDesc {
   }
 
   public String getInputFormat() {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return createTblDesc.getInputFormat();
       case VIEW:
@@ -253,7 +251,7 @@ public class ImportTableDesc {
   }
 
   public String getOutputFormat() {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return createTblDesc.getOutputFormat();
       case VIEW:
@@ -263,7 +261,7 @@ public class ImportTableDesc {
   }
 
   public String getSerName() {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return createTblDesc.getSerName();
       case VIEW:
@@ -273,7 +271,7 @@ public class ImportTableDesc {
   }
 
   public Map<String, String> getSerdeProps() {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return createTblDesc.getSerdeProps();
       case VIEW:
@@ -283,14 +281,14 @@ public class ImportTableDesc {
   }
 
   public List<String> getBucketCols() {
-    if (TYPE.TABLE.equals(getTableType())) {
+    if (TYPE.TABLE.equals(getDescType())) {
       return createTblDesc.getBucketCols();
     }
     return null;
   }
 
   public List<Order> getSortCols() {
-    if (TYPE.TABLE.equals(getTableType())) {
+    if (TYPE.TABLE.equals(getDescType())) {
       return createTblDesc.getSortCols();
     }
     return null;
@@ -300,7 +298,7 @@ public class ImportTableDesc {
    * @param replaceMode Determine if this CreateTable should behave like a replace-into alter instead
    */
   public void setReplaceMode(boolean replaceMode) {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         createTblDesc.setReplaceMode(replaceMode);
         break;
@@ -315,7 +313,7 @@ public class ImportTableDesc {
 
   public Task<? extends Serializable> getCreateTableTask(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
       HiveConf conf) {
-    switch (getTableType()) {
+    switch (getDescType()) {
       case TABLE:
         return TaskFactory.get(new DDLWork(inputs, outputs, createTblDesc), conf);
       case VIEW:
@@ -332,4 +330,13 @@ public class ImportTableDesc {
   public boolean isMaterializedView() {
     return table.isMaterializedView();
   }
+
+  public TableType tableType() {
+    if (isView()) {
+      return TableType.VIRTUAL_VIEW;
+    } else if (isMaterializedView()) {
+      return TableType.MATERIALIZED_VIEW;
+    }
+    return TableType.MANAGED_TABLE;
+  }
 }


Mime
View raw message