sentry-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sra...@apache.org
Subject [2/2] sentry git commit: Sentry-1321: Implement HMSFollower in Sentry service which reads the NotificationLog entries (Sravya Tirukkovalur, Reviewed by: Hao Hao) (Append)
Date Tue, 16 Aug 2016 00:26:09 GMT
Sentry-1321: Implement HMSFollower in Sentry service which reads the NotificationLog entries (Sravya Tirukkovalur, Reviewed by: Hao Hao) (Append)

Change-Id: If6e3d4a754b71cc0ba0cec1e34a44e6d9816ea38


Project: http://git-wip-us.apache.org/repos/asf/sentry/repo
Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/908072d6
Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/908072d6
Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/908072d6

Branch: refs/heads/sentry-ha-redesign
Commit: 908072d66bbabd91491a7d8fe8d9fa558dbc9c46
Parents: 2d5ed99
Author: Sravya Tirukkovalur <sravya@apache.org>
Authored: Mon Aug 15 17:25:47 2016 -0700
Committer: Sravya Tirukkovalur <sravya@apache.org>
Committed: Mon Aug 15 17:25:47 2016 -0700

----------------------------------------------------------------------
 sentry-binding/sentry-binding-hive-conf/pom.xml |  62 +++
 .../sentry/binding/hive/conf/HiveAuthzConf.java | 269 ++++++++++++
 .../sentry-binding-hive-follower/pom.xml        |  71 ++++
 .../json/JSONAlterPartitionMessage.java         |  78 ++++
 .../messaging/json/JSONAlterTableMessage.java   |  68 +++
 .../json/SentryJSONAddPartitionMessage.java     |  49 +++
 .../json/SentryJSONAlterPartitionMessage.java   |  53 +++
 .../json/SentryJSONAlterTableMessage.java       |  50 +++
 .../json/SentryJSONCreateDatabaseMessage.java   |  44 ++
 .../json/SentryJSONCreateTableMessage.java      |  45 ++
 .../json/SentryJSONDropDatabaseMessage.java     |  44 ++
 .../json/SentryJSONDropPartitionMessage.java    |  49 +++
 .../json/SentryJSONDropTableMessage.java        |  45 ++
 .../json/SentryJSONMessageDeserializer.java     | 110 +++++
 .../json/SentryJSONMessageFactory.java          | 177 ++++++++
 .../SentryInvalidHMSEventException.java         |  31 ++
 .../exception/SentryStandbyException.java       |   4 +
 .../SentryPolicyServiceClientDefaultImpl.java   |  12 +-
 .../sentry/service/thrift/HMSFollower.java      | 411 +++++++++++++++++++
 .../thrift/PoolClientInvocationHandler.java     | 305 +++++++++++---
 .../thrift/SentryServiceClientFactory.java      |  11 +-
 .../thrift/SentryServiceClientPoolFactory.java  |  11 +-
 .../thrift/TestSentryServiceFailureCase.java    |   5 +-
 .../TestSentryServiceWithInvalidMsgSize.java    |   2 +-
 .../sentry/service/thrift/TestHMSFollower.java  | 165 ++++++++
 .../thrift/TestPoolClientInvocationHandler.java |  69 ++++
 26 files changed, 2173 insertions(+), 67 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-conf/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-conf/pom.xml b/sentry-binding/sentry-binding-hive-conf/pom.xml
new file mode 100644
index 0000000..2cae541
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-conf/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.sentry</groupId>
+    <artifactId>sentry-binding</artifactId>
+    <version>1.8.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>sentry-binding-hive-conf</artifactId>
+  <name>Sentry Hive Binding Configuration</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.thrift</groupId>
+      <artifactId>libthrift</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.derby</groupId>
+      <artifactId>derby</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-service</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+
+</project>

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
new file mode 100644
index 0000000..4de755f
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-conf/src/main/java/org/apache/sentry/binding/hive/conf/HiveAuthzConf.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.hive.conf;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class HiveAuthzConf extends Configuration {
+
+    /**
+     * Configuration key used in hive-site.xml to point at sentry-site.xml
+     */
+    public static final String HIVE_ACCESS_CONF_URL = "hive.access.conf.url";
+    public static final String HIVE_SENTRY_CONF_URL = "hive.sentry.conf.url";
+    public static final String HIVE_ACCESS_SUBJECT_NAME = "hive.access.subject.name";
+    public static final String HIVE_SENTRY_SUBJECT_NAME = "hive.sentry.subject.name";
+    public static final String HIVE_SENTRY_AUTH_ERRORS = "sentry.hive.authorization.errors";
+    public static final String HIVE_SENTRY_MOCK_COMPILATION = "sentry.hive.mock.compilation";
+    public static final String HIVE_SENTRY_MOCK_ERROR = "sentry.hive.mock.error";
+    public static final String HIVE_SENTRY_PRIVILEGE_ERROR_MESSAGE = "No valid privileges";
+    /**
+     * Property used to persist the role set in the session. This is not public for now.
+     */
+    public static final String SENTRY_ACTIVE_ROLE_SET = "hive.sentry.active.role.set";
+
+    public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST =
+            "hive.sentry.security.command.whitelist";
+    public static final String HIVE_SENTRY_SECURITY_COMMAND_WHITELIST_DEFAULT =
+            "set,reset,reload";
+
+    public static final String HIVE_SENTRY_SERDE_WHITELIST = "hive.sentry.serde.whitelist";
+    public static final String HIVE_SENTRY_SERDE_WHITELIST_DEFAULT = "org.apache.hadoop.hive.serde2";
+
+    // Disable the serde Uri privileges by default for backward compatibilities.
+    public static final String HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED = "hive.sentry.turn.on.serde.uri.privileges";
+    public static final boolean HIVE_SENTRY_SERDE_URI_PRIVILIEGES_ENABLED_DEFAULT = false;
+
+    public static final String HIVE_UDF_WHITE_LIST =
+            "concat,substr,substring,space,repeat,ascii,lpad,rpad,size,round,floor,sqrt,ceil," +
+                    "ceiling,rand,abs,pmod,ln,log2,sin,asin,cos,acos,log10,log,exp,power,pow,sign,pi," +
+                    "degrees,radians,atan,tan,e,conv,bin,hex,unhex,base64,unbase64,encode,decode,upper," +
+                    "lower,ucase,lcase,trim,ltrim,rtrim,length,reverse,field,find_in_set,initcap,like," +
+                    "rlike,regexp,regexp_replace,regexp_extract,parse_url,nvl,split,str_to_map,translate" +
+                    ",positive,negative,day,dayofmonth,month,year,hour,minute,second,from_unixtime," +
+                    "to_date,weekofyear,last_day,date_add,date_sub,datediff,add_months,get_json_object," +
+                    "xpath_string,xpath_boolean,xpath_number,xpath_double,xpath_float,xpath_long," +
+                    "xpath_int,xpath_short,xpath,+,-,*,/,%,div,&,|,^,~,current_database,isnull," +
+                    "isnotnull,if,in,and,or,=,==,<=>,!=,<>,<,<=,>,>=,not,!,between,ewah_bitmap_and," +
+                    "ewah_bitmap_or,ewah_bitmap_empty,boolean,tinyint,smallint,int,bigint,float,double," +
+                    "string,date,timestamp,binary,decimal,varchar,char,max,min,sum,count,avg,std,stddev," +
+                    "stddev_pop,stddev_samp,variance,var_pop,var_samp,covar_pop,covar_samp,corr," +
+                    "histogram_numeric,percentile_approx,collect_set,collect_list,ngrams," +
+                    "context_ngrams,ewah_bitmap,compute_stats,percentile," +
+                    "array,assert_true,map,struct,named_struct,create_union,case,when,hash,coalesce," +
+                    "index,in_file,instr,locate,elt,concat_ws,sort_array," +
+                    "array_contains,sentences,map_keys,map_values,format_number,printf,greatest,least," +
+                    "from_utc_timestamp,to_utc_timestamp,unix_timestamp,to_unix_timestamp,explode," +
+                    "inline,json_tuple,parse_url_tuple,posexplode,stack,lead,lag,row_number,rank," +
+                    "dense_rank,percent_rank,cume_dist,ntile,first_value,last_value,noop,noopwithmap," +
+                    "noopstreaming,noopwithmapstreaming,windowingtablefunction,matchpath";
+
+    public static final String HIVE_UDF_BLACK_LIST = "reflect,reflect2,java_method";
+
+    /**
+     * Config setting definitions
+     */
+    public static enum AuthzConfVars {
+        AUTHZ_PROVIDER("sentry.provider",
+                "org.apache.sentry.provider.common.HadoopGroupResourceAuthorizationProvider"),
+        AUTHZ_PROVIDER_RESOURCE("sentry.hive.provider.resource", ""),
+        AUTHZ_PROVIDER_BACKEND("sentry.hive.provider.backend", "org.apache.sentry.provider.file.SimpleFileProviderBackend"),
+        AUTHZ_POLICY_ENGINE("sentry.hive.policy.engine", "org.apache.sentry.policy.engine.common.CommonPolicyEngine"),
+        AUTHZ_POLICY_FILE_FORMATTER(
+                "sentry.hive.policy.file.formatter",
+                "org.apache.sentry.binding.hive.SentryIniPolicyFileFormatter"),
+        AUTHZ_SERVER_NAME("sentry.hive.server", ""),
+        AUTHZ_RESTRICT_DEFAULT_DB("sentry.hive.restrict.defaultDB", "false"),
+        SENTRY_TESTING_MODE("sentry.hive.testing.mode", "false"),
+        AUTHZ_ALLOW_HIVE_IMPERSONATION("sentry.hive.allow.hive.impersonation", "false"),
+        AUTHZ_ONFAILURE_HOOKS("sentry.hive.failure.hooks", ""),
+        AUTHZ_METASTORE_SERVICE_USERS("sentry.metastore.service.users", null),
+        AUTHZ_SYNC_ALTER_WITH_POLICY_STORE("sentry.hive.sync.alter", "true"),
+        AUTHZ_SYNC_CREATE_WITH_POLICY_STORE("sentry.hive.sync.create", "false"),
+        AUTHZ_SYNC_DROP_WITH_POLICY_STORE("sentry.hive.sync.drop", "true"),
+
+        AUTHZ_PROVIDER_DEPRECATED("hive.sentry.provider",
+                "org.apache.sentry.provider.file.ResourceAuthorizationProvider"),
+        AUTHZ_PROVIDER_RESOURCE_DEPRECATED("hive.sentry.provider.resource", ""),
+        AUTHZ_SERVER_NAME_DEPRECATED("hive.sentry.server", ""),
+        AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED("hive.sentry.restrict.defaultDB", "false"),
+        SENTRY_TESTING_MODE_DEPRECATED("hive.sentry.testing.mode", "false"),
+        AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED("hive.sentry.allow.hive.impersonation", "false"),
+        AUTHZ_ONFAILURE_HOOKS_DEPRECATED("hive.sentry.failure.hooks", "");
+
+        private final String varName;
+        private final String defaultVal;
+
+        AuthzConfVars(String varName, String defaultVal) {
+            this.varName = varName;
+            this.defaultVal = defaultVal;
+        }
+
+        public String getVar() {
+            return varName;
+        }
+
+        public String getDefault() {
+            return defaultVal;
+        }
+
+        public static String getDefault(String varName) {
+            for (AuthzConfVars oneVar : AuthzConfVars.values()) {
+                if(oneVar.getVar().equalsIgnoreCase(varName)) {
+                    return oneVar.getDefault();
+                }
+            }
+            return null;
+        }
+    }
+
+    // map of current property names - > deprecated property names.
+    // The binding layer code should work if the deprecated property names are provided,
+    // as long as the new property names aren't also provided.  Since the binding code
+    // only calls the new property names, we require a map from current names to deprecated
+    // names in order to check if the deprecated name of a property was set.
+    private static final Map<String, AuthzConfVars> currentToDeprecatedProps =
+            new HashMap<String, AuthzConfVars>();
+    static {
+        currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER.getVar(), AuthzConfVars.AUTHZ_PROVIDER_DEPRECATED);
+        currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_PROVIDER_RESOURCE.getVar(), AuthzConfVars.AUTHZ_PROVIDER_RESOURCE_DEPRECATED);
+        currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_SERVER_NAME.getVar(), AuthzConfVars.AUTHZ_SERVER_NAME_DEPRECATED);
+        currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB.getVar(), AuthzConfVars.AUTHZ_RESTRICT_DEFAULT_DB_DEPRECATED);
+        currentToDeprecatedProps.put(AuthzConfVars.SENTRY_TESTING_MODE.getVar(), AuthzConfVars.SENTRY_TESTING_MODE_DEPRECATED);
+        currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION.getVar(), AuthzConfVars.AUTHZ_ALLOW_HIVE_IMPERSONATION_DEPRECATED);
+        currentToDeprecatedProps.put(AuthzConfVars.AUTHZ_ONFAILURE_HOOKS.getVar(), AuthzConfVars.AUTHZ_ONFAILURE_HOOKS_DEPRECATED);
+    };
+
+    private static final Logger LOG = LoggerFactory
+            .getLogger(HiveAuthzConf.class);
+    public static final String AUTHZ_SITE_FILE = "sentry-site.xml";
+    private final String hiveAuthzSiteFile;
+
+    public HiveAuthzConf(URL hiveAuthzSiteURL) {
+        super();
+        LOG.info("DefaultFS: " + super.get("fs.defaultFS"));
+        addResource(hiveAuthzSiteURL);
+        applySystemProperties();
+        LOG.info("DefaultFS: " + super.get("fs.defaultFS"));
+        this.hiveAuthzSiteFile = hiveAuthzSiteURL.toString();
+    }
+    /**
+     * Apply system properties to this object if the property name is defined in ConfVars
+     * and the value is non-null and not an empty string.
+     */
+    private void applySystemProperties() {
+        Map<String, String> systemProperties = getConfSystemProperties();
+        for (Entry<String, String> systemProperty : systemProperties.entrySet()) {
+            this.set(systemProperty.getKey(), systemProperty.getValue());
+        }
+    }
+
+    /**
+     * This method returns a mapping from config variable name to its value for all config variables
+     * which have been set using System properties
+     */
+    public static Map<String, String> getConfSystemProperties() {
+        Map<String, String> systemProperties = new HashMap<String, String>();
+
+        for (AuthzConfVars oneVar : AuthzConfVars.values()) {
+            String value = System.getProperty(oneVar.getVar());
+            if (value != null && value.length() > 0) {
+                systemProperties.put(oneVar.getVar(), value);
+            }
+        }
+        return systemProperties;
+    }
+
+    @Override
+    public String get(String varName) {
+        return get(varName, null);
+    }
+
+    @Override
+    public String get(String varName, String defaultVal) {
+        String retVal = super.get(varName);
+        if (retVal == null) {
+            // check if the deprecated value is set here
+            if (currentToDeprecatedProps.containsKey(varName)) {
+                retVal = super.get(currentToDeprecatedProps.get(varName).getVar());
+            }
+            if (retVal == null) {
+                retVal = AuthzConfVars.getDefault(varName);
+            } else {
+                LOG.warn("Using the deprecated config setting " + currentToDeprecatedProps.get(varName).getVar() +
+                        " instead of " + varName);
+            }
+        }
+        if (retVal == null) {
+            retVal = defaultVal;
+        }
+        return retVal;
+    }
+
+    public String getHiveAuthzSiteFile() {
+        return hiveAuthzSiteFile;
+    }
+
+    /**
+     * Extract the authz config file path from given hive conf and load the authz config
+     * @param hiveConf
+     * @return
+     * @throws IllegalArgumentException
+     */
+    public static HiveAuthzConf getAuthzConf(HiveConf hiveConf)
+            throws IllegalArgumentException {
+        boolean depreicatedConfigFile = false;
+
+        String hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_SENTRY_CONF_URL);
+        if (hiveAuthzConf == null
+                || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) {
+            hiveAuthzConf = hiveConf.get(HiveAuthzConf.HIVE_ACCESS_CONF_URL);
+            depreicatedConfigFile = true;
+        }
+
+        if (hiveAuthzConf == null
+                || (hiveAuthzConf = hiveAuthzConf.trim()).isEmpty()) {
+            throw new IllegalArgumentException("Configuration key "
+                    + HiveAuthzConf.HIVE_SENTRY_CONF_URL + " value '" + hiveAuthzConf
+                    + "' is invalid.");
+        }
+
+        try {
+            return new HiveAuthzConf(new URL(hiveAuthzConf));
+        } catch (MalformedURLException e) {
+            if (depreicatedConfigFile) {
+                throw new IllegalArgumentException("Configuration key "
+                        + HiveAuthzConf.HIVE_ACCESS_CONF_URL
+                        + " specifies a malformed URL '" + hiveAuthzConf + "'", e);
+            } else {
+                throw new IllegalArgumentException("Configuration key "
+                        + HiveAuthzConf.HIVE_SENTRY_CONF_URL
+                        + " specifies a malformed URL '" + hiveAuthzConf + "'", e);
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/pom.xml
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/pom.xml b/sentry-binding/sentry-binding-hive-follower/pom.xml
new file mode 100644
index 0000000..bcfb417
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.sentry</groupId>
+    <artifactId>sentry-binding</artifactId>
+    <version>1.8.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>sentry-binding-hive-follower</artifactId>
+  <name>Hive follower for Sentry</name>
+
+  <properties>
+    <datanucleus-api-jdo.version>3.2.6</datanucleus-api-jdo.version>
+    <datanucleus-core.version>3.2.12</datanucleus-core.version>
+    <datanucleus-rdbms.version>3.2.12</datanucleus-rdbms.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-core</artifactId>
+      <version>${datanucleus-core.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-api-jdo</artifactId>
+      <version>${datanucleus-api-jdo.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.datanucleus</groupId>
+      <artifactId>datanucleus-rdbms</artifactId>
+      <version>${datanucleus-rdbms.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive.hcatalog</groupId>
+      <artifactId>hive-hcatalog-server-extensions</artifactId>
+      <version>${hive.version}</version>
+      <scope>compile</scope>
+    </dependency>
+  </dependencies>
+
+</project>

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
new file mode 100644
index 0000000..890186b
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterPartitionMessage.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.AlterPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+/*
+* This is only needed as corresponding class in Hive 1.1.0 does not have a default constructor
+ */
+public class JSONAlterPartitionMessage extends AlterPartitionMessage {
+    @JsonProperty
+    String server;
+    @JsonProperty
+    String servicePrincipal;
+    @JsonProperty
+    String db;
+    @JsonProperty
+    String table;
+    @JsonProperty
+    Long timestamp;
+    @JsonProperty
+    List<String> values;
+
+    public JSONAlterPartitionMessage() {}
+    public JSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table, List<String> values, Long timestamp) {
+        this.server = server;
+        this.servicePrincipal = servicePrincipal;
+        this.db = db;
+        this.table = table;
+        this.timestamp = timestamp;
+        this.values = values;
+        this.checkValid();
+    }
+
+    public String getServer() {
+        return this.server;
+    }
+
+    public String getServicePrincipal() {
+        return this.servicePrincipal;
+    }
+
+    public String getDB() {
+        return this.db;
+    }
+
+    public Long getTimestamp() {
+        return this.timestamp;
+    }
+
+    public String getTable() {
+        return this.table;
+    }
+
+    public List<String> getValues() {
+        return this.values;
+    }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
new file mode 100644
index 0000000..76211c3
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/JSONAlterTableMessage.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.AlterTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+/**
+ * This class is required as this class does not have a default contructor in Hive 1.1.0
+ */
+public class JSONAlterTableMessage extends AlterTableMessage {
+    @JsonProperty
+    String server;
+    @JsonProperty
+    String servicePrincipal;
+    @JsonProperty
+    String db;
+    @JsonProperty
+    String table;
+    @JsonProperty
+    Long timestamp;
+
+    public JSONAlterTableMessage() {}
+    public JSONAlterTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp) {
+        this.server = server;
+        this.servicePrincipal = servicePrincipal;
+        this.db = db;
+        this.table = table;
+        this.timestamp = timestamp;
+        this.checkValid();
+    }
+
+    public String getServer() {
+        return this.server;
+    }
+
+    public String getServicePrincipal() {
+        return this.servicePrincipal;
+    }
+
+    public String getDB() {
+        return this.db;
+    }
+
+    public Long getTimestamp() {
+        return this.timestamp;
+    }
+
+    public String getTable() {
+        return this.table;
+    }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
new file mode 100644
index 0000000..c0c469c
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAddPartitionMessage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONAddPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+public class SentryJSONAddPartitionMessage extends JSONAddPartitionMessage {
+    @JsonProperty
+    List<String> locations;
+
+    public SentryJSONAddPartitionMessage() {
+    }
+
+    public SentryJSONAddPartitionMessage(String server, String servicePrincipal, String db, String table,
+                                         List<Map<String, String>> partitions, Long timestamp, List<String> locations) {
+        super(server, servicePrincipal, db, table, partitions, timestamp);
+        this.locations = locations;
+    }
+
+    public List<String> getLocations() {
+        return locations;
+    }
+
+    @Override
+    public String toString() {
+        return SentryJSONMessageDeserializer.serialize(this);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
new file mode 100644
index 0000000..99eb67a
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterPartitionMessage.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+public class SentryJSONAlterPartitionMessage extends JSONAlterPartitionMessage{
+    @JsonProperty
+    String location;
+    @JsonProperty
+    String oldLocation;
+
+    public SentryJSONAlterPartitionMessage() {
+    }
+
+    public SentryJSONAlterPartitionMessage(String server, String servicePrincipal, String db, String table,
+                                           List<String> values, Long timestamp, String oldlocation, String newLocation) {
+        super(server, servicePrincipal, db, table, values, timestamp);
+        this.location = newLocation;
+        this.oldLocation = oldlocation;
+    }
+
+    public String getLocation() {
+        return location;
+    }
+
+    public String getOldLocation() {
+        return oldLocation;
+    }
+
+    @Override
+    public String toString() {
+        return SentryJSONMessageDeserializer.serialize(this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
new file mode 100644
index 0000000..6e59e25
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONAlterTableMessage.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONAlterTableMessage extends JSONAlterTableMessage {
+    @JsonProperty
+    String location; //newLocation
+    @JsonProperty
+    String oldLocation;
+
+    public SentryJSONAlterTableMessage() {
+    }
+
+    public SentryJSONAlterTableMessage(String server, String servicePrincipal, String db, String table,
+                                       Long timestamp, String oldLocation, String location) {
+        super(server, servicePrincipal, db, table, timestamp);
+        this.location = location;
+        this.oldLocation = oldLocation;
+    }
+
+    public String getLocation() {
+        return location;
+    }
+    public String getOldLocation() {
+        return oldLocation;
+    }
+
+    @Override
+    public String toString() {
+        return SentryJSONMessageDeserializer.serialize(this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
new file mode 100644
index 0000000..ba19cbe
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateDatabaseMessage.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONCreateDatabaseMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONCreateDatabaseMessage extends JSONCreateDatabaseMessage {
+    @JsonProperty
+    String location;
+
+    public SentryJSONCreateDatabaseMessage() {
+    }
+
+    public SentryJSONCreateDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) {
+        super(server, servicePrincipal, db, timestamp);
+        this.location = location;
+    }
+
+    public String getLocation() {
+        return location;
+    }
+
+    @Override
+    public String toString() {
+        return SentryJSONMessageDeserializer.serialize(this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
new file mode 100644
index 0000000..57d11d2
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONCreateTableMessage.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONCreateTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONCreateTableMessage extends JSONCreateTableMessage {
+    @JsonProperty
+    String location;
+
+    public SentryJSONCreateTableMessage() {
+    }
+
+    public SentryJSONCreateTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) {
+        super(server, servicePrincipal, db, table, timestamp);
+        this.location = location;
+    }
+
+    public String getLocation() {
+        return location;
+    }
+
+    @Override
+    public String toString() {
+        return SentryJSONMessageDeserializer.serialize(this);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
new file mode 100644
index 0000000..05f83f7
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropDatabaseMessage.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropDatabaseMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+public class SentryJSONDropDatabaseMessage extends JSONDropDatabaseMessage{
+    @JsonProperty
+    String location;
+
+    public SentryJSONDropDatabaseMessage() {
+    }
+
+    public SentryJSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp, String location) {
+        super(server, servicePrincipal, db, timestamp);
+        this.location = location;
+    }
+
+    public String getLocation() {
+        return location;
+    }
+
+    @Override
+    public String toString() {
+        return SentryJSONMessageDeserializer.serialize(this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
new file mode 100644
index 0000000..2ab61f7
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropPartitionMessage.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropPartitionMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+import java.util.Map;
+
+public class SentryJSONDropPartitionMessage extends JSONDropPartitionMessage {
+    @JsonProperty
+    String location;
+
+    public SentryJSONDropPartitionMessage() {
+    }
+
+    public SentryJSONDropPartitionMessage(String server, String servicePrincipal, String db, String table,
+                                          List<Map<String, String>> partitions, Long timestamp, String location) {
+        super(server, servicePrincipal, db, table, partitions, timestamp);
+        this.location = location;
+    }
+
+    public String getLocation() {
+        return location;
+    }
+
+    @Override
+    public String toString() {
+        return SentryJSONMessageDeserializer.serialize(this);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
new file mode 100644
index 0000000..7005776
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONDropTableMessage.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.json.JSONDropTableMessage;
+import org.codehaus.jackson.annotate.JsonProperty;
+
+
+public class SentryJSONDropTableMessage extends JSONDropTableMessage {
+    @JsonProperty
+    String location;
+
+    public SentryJSONDropTableMessage() {
+    }
+
+    public SentryJSONDropTableMessage(String server, String servicePrincipal, String db, String table, Long timestamp, String location) {
+        super(server, servicePrincipal, db, table, timestamp);
+        this.location = location;
+    }
+
+    public String getLocation() {
+        return location;
+    }
+
+    @Override
+    public String toString() {
+        return SentryJSONMessageDeserializer.serialize(this);
+    }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
new file mode 100644
index 0000000..b645c45
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageDeserializer.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.hive.hcatalog.messaging.*;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.ObjectMapper;
+
+public class SentryJSONMessageDeserializer extends MessageDeserializer {
+    static ObjectMapper mapper = new ObjectMapper();
+
+    public SentryJSONMessageDeserializer() {
+    }
+
+    /**
+     * Method to de-serialize CreateDatabaseMessage instance.
+     */
+    public SentryJSONCreateDatabaseMessage getCreateDatabaseMessage(String messageBody) {
+        try {
+            return (SentryJSONCreateDatabaseMessage)mapper.readValue(messageBody, SentryJSONCreateDatabaseMessage.class);
+        } catch (Exception var3) {
+            throw new IllegalArgumentException("Could not construct SentryJSONCreateDatabaseMessage.", var3);
+        }
+    }
+
+    public SentryJSONDropDatabaseMessage getDropDatabaseMessage(String messageBody) {
+        try {
+            return (SentryJSONDropDatabaseMessage)mapper.readValue(messageBody, SentryJSONDropDatabaseMessage.class);
+        } catch (Exception var3) {
+            throw new IllegalArgumentException("Could not construct SentryJSONDropDatabaseMessage.", var3);
+        }
+    }
+
+    public SentryJSONCreateTableMessage getCreateTableMessage(String messageBody) {
+        try {
+            return (SentryJSONCreateTableMessage)mapper.readValue(messageBody, SentryJSONCreateTableMessage.class);
+        } catch (Exception var3) {
+            throw new IllegalArgumentException("Could not construct SentryJSONCreateTableMessage.", var3);
+        }
+    }
+
+    public SentryJSONAlterTableMessage getAlterTableMessage(String messageBody) {
+        try {
+            return (SentryJSONAlterTableMessage)mapper.readValue(messageBody, SentryJSONAlterTableMessage.class);
+        } catch (Exception var3) {
+            throw new IllegalArgumentException("Could not construct SentryJSONAlterTableMessage.", var3);
+        }
+    }
+
+    public SentryJSONDropTableMessage getDropTableMessage(String messageBody) {
+        try {
+            return (SentryJSONDropTableMessage)mapper.readValue(messageBody, SentryJSONDropTableMessage.class);
+        } catch (Exception var3) {
+            throw new IllegalArgumentException("Could not construct SentryJSONDropTableMessage.", var3);
+        }
+    }
+
+    public SentryJSONAddPartitionMessage getAddPartitionMessage(String messageBody) {
+        try {
+            return (SentryJSONAddPartitionMessage)mapper.readValue(messageBody, SentryJSONAddPartitionMessage.class);
+        } catch (Exception var3) {
+            throw new IllegalArgumentException("Could not construct SentryJSONAddPartitionMessage.", var3);
+        }
+    }
+
+    public SentryJSONAlterPartitionMessage getAlterPartitionMessage(String messageBody) {
+        try {
+            return (SentryJSONAlterPartitionMessage)mapper.readValue(messageBody, SentryJSONAlterPartitionMessage.class);
+        } catch (Exception var3) {
+            throw new IllegalArgumentException("Could not construct SentryJSONAlterPartitionMessage.", var3);
+        }
+    }
+
+    public SentryJSONDropPartitionMessage getDropPartitionMessage(String messageBody) {
+        try {
+            return (SentryJSONDropPartitionMessage)mapper.readValue(messageBody, SentryJSONDropPartitionMessage.class);
+        } catch (Exception var3) {
+            throw new IllegalArgumentException("Could not construct SentryJSONDropPartitionMessage.", var3);
+        }
+    }
+
+    static {
+        mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    }
+
+    public static String serialize(Object object) {
+        try {
+            return mapper.writeValueAsString(object);
+        }
+        catch (Exception exception) {
+            throw new IllegalArgumentException("Could not serialize: ", exception);
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
----------------------------------------------------------------------
diff --git a/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
new file mode 100644
index 0000000..00e7db8
--- /dev/null
+++ b/sentry-binding/sentry-binding-hive-follower/src/main/java/org/apache/sentry/binding/metastore/messaging/json/SentryJSONMessageFactory.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.binding.metastore.messaging.json;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
+import org.apache.hive.hcatalog.messaging.*;
+
+import java.util.*;
+
+public class SentryJSONMessageFactory extends MessageFactory {
+    private static final Log LOG = LogFactory.getLog(SentryJSONMessageFactory.class.getName());
+    private static SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
+    public SentryJSONMessageFactory() {
+        LOG.info("Using SentryJSONMessageFactory for building Notification log messages ");
+
+    }
+    public MessageDeserializer getDeserializer() {
+        return deserializer;
+    }
+
+    public String getVersion() {
+        return "0.1";
+    }
+
+    public String getMessageFormat() {
+        return "json";
+    }
+
+    public SentryJSONCreateDatabaseMessage buildCreateDatabaseMessage(Database db) {
+        return new SentryJSONCreateDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
+                Long.valueOf(this.now()), db.getLocationUri());
+    }
+    public SentryJSONDropDatabaseMessage buildDropDatabaseMessage(Database db) {
+        return new SentryJSONDropDatabaseMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, db.getName(),
+                Long.valueOf(this.now()), db.getLocationUri());
+    }
+
+    public SentryJSONCreateTableMessage buildCreateTableMessage(Table table) {
+        return new SentryJSONCreateTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+                table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation());
+    }
+
+    public SentryJSONAlterTableMessage buildAlterTableMessage(Table before, Table after) {
+        return new SentryJSONAlterTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(),
+                before.getTableName(), Long.valueOf(this.now()), before.getSd().getLocation(), after.getSd().getLocation());
+    }
+
+    public SentryJSONDropTableMessage buildDropTableMessage(Table table) {
+        return new SentryJSONDropTableMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+                table.getTableName(), Long.valueOf(this.now()), table.getSd().getLocation());
+    }
+
+    public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, List<Partition> partitions) {
+        return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+                table.getTableName(), getPartitionKeyValues(table, partitions), Long.valueOf(this.now()),
+                getPartitionLocations(partitions));
+    }
+
+    private List<String> getPartitionLocations(List<Partition> partitions) {
+        List<String> paths = new ArrayList<String>();
+        for(Partition partition:partitions) {
+            paths.add(partition.getSd().getLocation());
+        }
+        return paths;
+    }
+
+    //TODO: Not sure what is this used for. Need to investigate
+    private List<String> getPartitionLocations(PartitionSpecProxy partitionSpec) {
+        Iterator<Partition> iterator = partitionSpec.getPartitionIterator();
+        List<String> locations = new ArrayList<String>();
+        while(iterator.hasNext()) {
+            locations.add(iterator.next().getSd().getLocation());
+        }
+        return locations;
+    }
+
+    @InterfaceAudience.LimitedPrivate({"Hive"})
+    @InterfaceStability.Evolving
+    public SentryJSONAddPartitionMessage buildAddPartitionMessage(Table table, PartitionSpecProxy partitionSpec) {
+        return new SentryJSONAddPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, table.getDbName(),
+                table.getTableName(), getPartitionKeyValues(table, partitionSpec), Long.valueOf(this.now()),
+                getPartitionLocations(partitionSpec));
+    }
+
+    public SentryJSONAlterPartitionMessage buildAlterPartitionMessage(Partition before, Partition after) {
+        /*
+     f (partitionEvent.getOldPartition() != null) {
+      oldLoc = partitionEvent.getOldPartition().getSd().getLocation();
+    }
+    if (partitionEvent.getNewPartition() != null) {
+      newLoc = partitionEvent.getNewPartition().getSd().getLocation();
+    }
+
+    if ((oldLoc != null) && (newLoc != null) && (!oldLoc.equals(newLoc))) {
+      String authzObj =
+              partitionEvent.getOldPartition().getDbName() + "."
+                      + partitionEvent.getOldPartition().getTableName();
+      for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
+        plugin.renameAuthzObject(authzObj, oldLoc,
+                authzObj, newLoc);
+      }
+    }
+        * */
+        return new SentryJSONAlterPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, before.getDbName(),
+                before.getTableName(), before.getValues(), Long.valueOf(this.now()), before.getSd().getLocation(),
+                after.getSd().getLocation());
+    }
+
+    public SentryJSONDropPartitionMessage buildDropPartitionMessage(Table table, Partition partition) {
+        return new SentryJSONDropPartitionMessage(HCAT_SERVER_URL, HCAT_SERVICE_PRINCIPAL, partition.getDbName(),
+                partition.getTableName(), Arrays.asList(getPartitionKeyValues(table, partition)),
+                Long.valueOf(this.now()), partition.getSd().getLocation());
+    }
+
+    private static Map<String, String> getPartitionKeyValues(Table table, Partition partition) {
+        LinkedHashMap partitionKeys = new LinkedHashMap();
+
+        for(int i = 0; i < table.getPartitionKeysSize(); ++i) {
+            partitionKeys.put(((FieldSchema)table.getPartitionKeys().get(i)).getName(), partition.getValues().get(i));
+        }
+
+        return partitionKeys;
+    }
+
+    private static List<Map<String, String>> getPartitionKeyValues(Table table, List<Partition> partitions) {
+        ArrayList partitionList = new ArrayList(partitions.size());
+        Iterator i$ = partitions.iterator();
+
+        while(i$.hasNext()) {
+            Partition partition = (Partition)i$.next();
+            partitionList.add(getPartitionKeyValues(table, partition));
+        }
+
+        return partitionList;
+    }
+
+    @InterfaceAudience.LimitedPrivate({"Hive"})
+    @InterfaceStability.Evolving
+    private static List<Map<String, String>> getPartitionKeyValues(Table table, PartitionSpecProxy partitionSpec) {
+        ArrayList partitionList = new ArrayList();
+        PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator();
+
+        while(iterator.hasNext()) {
+            Partition partition = (Partition)iterator.next();
+            partitionList.add(getPartitionKeyValues(table, partition));
+        }
+
+        return partitionList;
+    }
+    //This is private in parent class
+    private long now() {
+        return System.currentTimeMillis() / 1000L;
+    }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java
----------------------------------------------------------------------
diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java
new file mode 100644
index 0000000..50fb59c
--- /dev/null
+++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryInvalidHMSEventException.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.core.common.exception;
+
+public class SentryInvalidHMSEventException extends SentryUserException {
+  private static final long serialVersionUID = 29620806553835L;
+  public SentryInvalidHMSEventException(String msg) {
+    super(msg);
+  }
+  public SentryInvalidHMSEventException(String msg, String reason) {
+    super(msg, reason);
+  }
+  public SentryInvalidHMSEventException(String msg, Throwable cause) {
+    super(msg, cause);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
----------------------------------------------------------------------
diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
index b2df699..da6cfce 100644
--- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
+++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/core/common/exception/SentryStandbyException.java
@@ -34,4 +34,8 @@ public class SentryStandbyException extends SentryUserException {
   public SentryStandbyException(String msg, String reason) {
     super(msg, reason);
   }
+
+  public SentryStandbyException(String msg, Throwable t) {
+    super(msg, t);
+  }
 }

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
index 1039e6e..4f42a51 100644
--- a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/provider/db/service/thrift/SentryPolicyServiceClientDefaultImpl.java
@@ -132,12 +132,18 @@ public class SentryPolicyServiceClientDefaultImpl implements SentryPolicyService
   }
 
   public SentryPolicyServiceClientDefaultImpl(Configuration conf) throws IOException {
+    this(Preconditions.checkNotNull(conf.get(ClientConfig.SERVER_RPC_ADDRESS), "Config key "
+        + ClientConfig.SERVER_RPC_ADDRESS + " is required"), conf.getInt(
+        ClientConfig.SERVER_RPC_PORT, ClientConfig.SERVER_RPC_PORT_DEFAULT), conf);
+  }
+
+  public SentryPolicyServiceClientDefaultImpl(String addr, int port,
+        Configuration conf) throws IOException {
     this.conf = conf;
     Preconditions.checkNotNull(this.conf, "Configuration object cannot be null");
     this.serverAddress = NetUtils.createSocketAddr(Preconditions.checkNotNull(
-                           conf.get(ClientConfig.SERVER_RPC_ADDRESS), "Config key "
-                           + ClientConfig.SERVER_RPC_ADDRESS + " is required"), conf.getInt(
-                           ClientConfig.SERVER_RPC_PORT, ClientConfig.SERVER_RPC_PORT_DEFAULT));
+                            addr, "Config key " + ClientConfig.SERVER_RPC_ADDRESS
+                            + " is required"), port);
     this.connectionTimeout = conf.getInt(ClientConfig.SERVER_RPC_CONN_TIMEOUT,
                                          ClientConfig.SERVER_RPC_CONN_TIMEOUT_DEFAULT);
     kerberos = ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase(

http://git-wip-us.apache.org/repos/asf/sentry/blob/908072d6/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
----------------------------------------------------------------------
diff --git a/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
new file mode 100644
index 0000000..4430471
--- /dev/null
+++ b/sentry-provider/sentry-provider-db/src/main/java/org/apache/sentry/service/thrift/HMSFollower.java
@@ -0,0 +1,411 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sentry.service.thrift;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hive.hcatalog.messaging.HCatEventMessage;
+import org.apache.sentry.binding.hive.conf.HiveAuthzConf;
+import org.apache.sentry.core.common.exception.*;
+import org.apache.sentry.provider.db.service.persistent.SentryStore;
+import org.apache.sentry.provider.db.service.thrift.TSentryAuthorizable;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.sentry.binding.metastore.messaging.json.*;
+
+import javax.security.auth.Subject;
+import javax.security.auth.login.LoginException;
+import java.io.File;
+import java.io.IOException;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
+
+import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE;
+import static org.apache.sentry.binding.hive.conf.HiveAuthzConf.AuthzConfVars.AUTHZ_SYNC_DROP_WITH_POLICY_STORE;
+
+/*
+HMSFollower is the thread which follows the Hive MetaStore state changes from Sentry.
+It gets the full update and notification logs from HMS and applies it to
+update permissions stored in Sentry using SentryStore and also update the <obj,path> state
+stored for HDFS- Sentry sync.
+ */
+@SuppressWarnings("PMD")
+public class HMSFollower implements Runnable {
+  private static final Logger LOGGER = LoggerFactory.getLogger(HMSFollower.class);
+
+  private long currentEventID;
+  private static boolean connectedToHMS = false;
+  private HiveMetaStoreClient client;
+  private SentryKerberosContext kerberosContext;
+  private Configuration authzConf;
+  private boolean kerberos;
+  private SentryStore sentryStore;
+  private String hiveInstance;
+  final static int maxRetriesForLogin = 3;
+  final static int maxRetriesForConnection = 3;
+
+  HMSFollower(Configuration conf) throws SentryNoSuchObjectException,
+      SentryAccessDeniedException, SentrySiteConfigurationException, IOException { //TODO: Handle any possible exceptions or throw specific exceptions
+    LOGGER.info("HMSFollower is being initialized");
+    authzConf = conf;
+    try {
+      sentryStore = new SentryStore(authzConf);
+    } catch (SentryStandbyException e) {
+      //TODO: Do not start HMSFollower if standby
+    }
+    //TODO: Initialize currentEventID from Sentry db
+    currentEventID = 0;
+  }
+
+  @VisibleForTesting
+  HMSFollower(Configuration conf, SentryStore sentryStore, String hiveInstance) {
+    this.authzConf = conf;
+    this.sentryStore = sentryStore;
+    this.hiveInstance = hiveInstance;
+  }
+
+  @VisibleForTesting
+  public static boolean isConnectedToHMS() {
+    return connectedToHMS;
+  }
+
+  /*
+  Returns HMS Client if successful, returns null if HMS is not ready yet to take connections
+  Throws @LoginException if Kerberos context creation failed using Sentry's kerberos credentials
+  Throws @MetaException if there was a problem on creating an HMSClient
+   */
+  private HiveMetaStoreClient getMetaStoreClient(Configuration conf)
+      throws LoginException, MetaException {
+    if(client != null) {
+      return client;
+    }
+    // Seems like HMS client creation although seems successful,
+    // it actually connects to an invalid HMS instance.
+    // So it seems like it is necessary to wait until we make sure metastore config is properly loaded.
+    boolean loadedHiveConf = HiveConf.isLoadMetastoreConfig();
+    if(!loadedHiveConf) {
+      return null;
+    }
+    final HiveConf hiveConf = new HiveConf();
+    hiveInstance = hiveConf.get(HiveAuthzConf.AuthzConfVars.AUTHZ_SERVER_NAME.getVar());
+
+    String principal, keytab;
+
+    //TODO: Is this the right(standard) way to create a HMS client? HiveMetastoreClientFactoryImpl?
+    //TODO: Check if HMS is using kerberos instead of relying on Sentry conf
+    //TODO: Handle TGT renewals
+    kerberos = ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS.equalsIgnoreCase(
+        conf.get(ServiceConstants.ServerConfig.SECURITY_MODE, ServiceConstants.ServerConfig.SECURITY_MODE_KERBEROS).trim());
+    if (kerberos) {
+      LOGGER.info("Making a kerberos connection to HMS");
+      //TODO: Is this needed? Use Hadoop libraries to translate the _HOST placeholder with actual hostname
+      //Validate principal
+      principal = Preconditions.checkNotNull(ServiceConstants.ServerConfig.PRINCIPAL,
+          ServiceConstants.ServerConfig.PRINCIPAL + " is required");
+      LOGGER.info("Using kerberos principal: " + principal);
+      final String[] principalParts = SaslRpcServer.splitKerberosName(principal);
+      Preconditions.checkArgument(principalParts.length == 3,
+          "Kerberos principal should have 3 parts: " + principal);
+
+      keytab = Preconditions.checkNotNull(conf.get(ServiceConstants.ServerConfig.KEY_TAB),
+          ServiceConstants.ServerConfig.KEY_TAB + " is required");
+      File keytabFile = new File(keytab);
+      Preconditions.checkState(keytabFile.isFile() && keytabFile.canRead(),
+          "Keytab " + keytab + " does not exist or is not readable.");
+      boolean establishedKerberosContext = false;
+      int attempt = 1;
+      while(establishedKerberosContext) {
+        try {
+          kerberosContext = new SentryKerberosContext(principal, keytab, true);
+          establishedKerberosContext = true;
+          LOGGER.info("Established kerberos context, will now connect to HMS");
+        } catch (LoginException e) {
+          //Kerberos login failed
+          if( attempt > maxRetriesForLogin ) {
+            throw e;
+          }
+          attempt++;
+        }
+      }
+      boolean establishedConnection = false;
+      attempt = 1;
+      while(establishedConnection) {
+        try {
+          client = Subject.doAs(kerberosContext.getSubject(), new PrivilegedExceptionAction<HiveMetaStoreClient>() {
+            @Override
+            public HiveMetaStoreClient run() throws Exception {
+              return new HiveMetaStoreClient(hiveConf);
+            }
+          });
+          LOGGER.info("Secure connection established with HMS");
+        } catch (PrivilegedActionException e) {
+          if( attempt > maxRetriesForConnection ) {
+            //We should just retry as it is possible that HMS is not ready yet to receive requests
+            //TODO: How do we differentiate between kerberos problem versus HMS not being up?
+            LOGGER.error("Cannot connect to HMS", e);
+          }
+          attempt++;
+        }
+      }
+    } else {
+      //This is only for testing purposes. Sentry strongly recommends strong authentication
+      client = new HiveMetaStoreClient(hiveConf);
+      LOGGER.info("Non secure connection established with HMS");
+    }
+    return client;
+  }
+
+  public void run() {
+    if( client == null ) {
+      try {
+        client = getMetaStoreClient(authzConf);
+        if (client == null) {
+          //TODO: Do we want to throw an exception after a certain timeout?
+          return;
+        } else {
+          connectedToHMS = true;
+          LOGGER.info("HMSFollower of Sentry successfully connected to HMS");
+        }
+      } catch (Exception e) {
+
+      }
+    }
+    if (needFullUpdate()) {
+      //TODO: Handle
+    } else {
+      try {
+        NotificationEventResponse response = client.getNextNotification(currentEventID, Integer.MAX_VALUE, null);
+        if (response.isSetEvents()) {
+          LOGGER.info(String.format("CurrentEventID = %s. Processing %s events",
+              currentEventID, response.getEvents().size()));
+          processNotificationEvents(response.getEvents());
+        }
+      } catch (TException e) {
+        LOGGER.error("ThriftException occured fetching Notification entries, will try");
+        e.printStackTrace();
+      } catch (SentryInvalidInputException|SentryInvalidHMSEventException e) {
+        throw new RuntimeException(e);
+      }
+    }
+  }
+
+  private boolean needFullUpdate() {
+    //TODO Implement
+    return false;
+  }
+
+  private boolean syncWithPolicyStore(HiveAuthzConf.AuthzConfVars syncConfVar) {
+    return "true"
+        .equalsIgnoreCase((authzConf.get(syncConfVar.getVar(), "true")));
+  }
+
+  /*
+  Throws SentryInvalidHMSEventException if Notification event contains insufficient information
+   */
+
+  void processNotificationEvents(List<NotificationEvent> events) throws
+      SentryInvalidHMSEventException, SentryInvalidInputException {
+    SentryJSONMessageDeserializer deserializer = new SentryJSONMessageDeserializer();
+
+    for (NotificationEvent event : events) {
+      String dbName, tableName, oldLocation, newLocation, location;
+      switch (HCatEventMessage.EventType.valueOf(event.getEventType())) {
+        case CREATE_DATABASE:
+          SentryJSONCreateDatabaseMessage message = deserializer.getCreateDatabaseMessage(event.getMessage());
+          dbName = message.getDB();
+
+          location = message.getLocation();
+          if (dbName == null || location == null) {
+            throw new SentryInvalidHMSEventException(String.format("Create database event has incomplete information. " +
+                "dbName = %s location = %s", dbName, location));
+          }
+          if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+            try {
+              dropSentryDbPrivileges(dbName);
+            } catch (SentryNoSuchObjectException e) {
+                LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName);
+            } catch (SentryInvalidInputException e) {
+              throw new SentryInvalidInputException("Could not process Create database event. Event: " + event.toString(), e);
+            }
+          }
+          //TODO: HDFSPlugin.addPath(dbName, location)
+          break;
+        case DROP_DATABASE:
+          SentryJSONDropDatabaseMessage dropDatabaseMessage = deserializer.getDropDatabaseMessage(event.getMessage());
+          dbName = dropDatabaseMessage.getDB();
+          if (dbName == null) {
+            throw new SentryInvalidHMSEventException(String.format("Drop database event has incomplete information. " +
+                "dbName = %s", dbName));
+          }
+          if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
+            try {
+              dropSentryDbPrivileges(dbName);
+            } catch (SentryNoSuchObjectException e) {
+              LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the database: %s", dbName);
+            } catch (SentryInvalidInputException e) {
+              throw new SentryInvalidInputException("Could not process Drop database event. Event: " + event.toString(), e);
+            }
+          }
+          //TODO: HDFSPlugin.deletePath(dbName, location)
+          break;
+        case CREATE_TABLE:
+          SentryJSONCreateTableMessage createTableMessage = deserializer.getCreateTableMessage(event.getMessage());
+          dbName = createTableMessage.getDB();
+          tableName = createTableMessage.getTable();
+          location = createTableMessage.getLocation();
+          if (dbName == null || tableName == null || location == null) {
+            throw new SentryInvalidHMSEventException(String.format("Create table event has incomplete information. " +
+                "dbName = %s, tableName = %s, location = %s", dbName, tableName, location));
+          }
+          if (syncWithPolicyStore(AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
+            try {
+              dropSentryTablePrivileges(dbName, tableName);
+            } catch (SentryNoSuchObjectException e) {
+              LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName);
+            } catch (SentryInvalidInputException e) {
+              throw new SentryInvalidInputException("Could not process Create table event. Event: " + event.toString(), e);
+            }
+          }
+          //TODO: HDFSPlugin.deletePath(dbName, location)
+          break;
+        case DROP_TABLE:
+          SentryJSONDropTableMessage dropTableMessage = deserializer.getDropTableMessage(event.getMessage());
+          dbName = dropTableMessage.getDB();
+          tableName = dropTableMessage.getTable();
+          if (dbName == null || tableName == null) {
+            throw new SentryInvalidHMSEventException(String.format("Drop table event has incomplete information. " +
+                "dbName = %s, tableName = %s", dbName, tableName));
+          }
+          if (syncWithPolicyStore(AUTHZ_SYNC_DROP_WITH_POLICY_STORE)) {
+            try{
+              dropSentryTablePrivileges(dbName, tableName);
+            } catch (SentryNoSuchObjectException e) {
+              LOGGER.info("Drop Sentry privilege ignored as there are no privileges on the table: %s.%s", dbName, tableName);
+            } catch (SentryInvalidInputException e) {
+              throw new SentryInvalidInputException("Could not process Drop table event. Event: " + event.toString(), e);
+            }
+          }
+          //TODO: HDFSPlugin.deletePath(dbName, location)
+          break;
+        case ALTER_TABLE:
+          SentryJSONAlterTableMessage alterTableMessage = deserializer.getAlterTableMessage(event.getMessage());
+
+          String oldDbName = alterTableMessage.getDB();
+          String oldTableName = alterTableMessage.getTable();
+          String newDbName = event.getDbName();
+          String newTableName = event.getTableName();
+          oldLocation = alterTableMessage.getOldLocation();
+          newLocation = alterTableMessage.getLocation();
+
+          if (oldDbName == null || oldTableName == null || newDbName == null || newTableName == null ||
+              oldLocation == null || newLocation == null) {
+            throw new SentryInvalidHMSEventException(String.format("Alter table event has incomplete information. " +
+                "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s",
+                oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation));
+          }
+
+          if(!newDbName.equalsIgnoreCase(oldDbName) || !oldTableName.equalsIgnoreCase(newTableName)) { // Name has changed
+            if(!oldLocation.equals(newLocation)) { // Location has changed
+
+              //Name and path has changed
+              // - Alter table rename for managed table
+              //TODO: Handle HDFS plugin
+
+            } else {
+              //Only name has changed
+              // - Alter table rename for an external table
+              //TODO: Handle HDFS plugin
+
+            }
+            try {
+              renamePrivileges(oldDbName, oldTableName, newDbName, newTableName);
+            } catch (SentryNoSuchObjectException e) {
+              LOGGER.info("Rename Sentry privilege ignored as there are no privileges on the table: %s.%s", oldDbName, oldTableName);
+            } catch (SentryInvalidInputException e) {
+              throw new SentryInvalidInputException("Could not process Alter table event. Event: " + event.toString(), e);
+            } catch (SentryStandbyException e) {
+              LOGGER.error("Seems like this process became a standby. " +
+                  "Ignoring the failure. The new leader will reprocess this notification event");
+            }
+          } else if(!oldLocation.equals(newLocation)) { // Only Location has changed{
+            //- Alter table set location
+            //TODO: Handle HDFS plugin
+          } else {
+            LOGGER.info(String.format("Alter table notification ignored as neither name nor location has changed: " +
+                "oldDbName = %s, oldTableName = %s, oldLocation = %s, newDbName = %s, newTableName = %s, newLocation = %s",
+            oldDbName, oldTableName, oldLocation, newDbName, newTableName, newLocation));
+          }
+          //TODO: Write test cases for all these cases
+          break;
+        case ADD_PARTITION:
+        case DROP_PARTITION:
+        case ALTER_PARTITION:
+          //TODO: Handle HDFS plugin
+          break;
+      }
+    currentEventID = event.getEventId();
+    }
+  }
+
+  private void dropSentryDbPrivileges(String dbName) throws SentryNoSuchObjectException, SentryInvalidInputException {
+    TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setDb(dbName);
+    try {
+      sentryStore.dropPrivilege(authorizable);
+    } catch (SentryStandbyException e) {
+      LOGGER.error("Seems like this process became a standby. " +
+          "Ignoring the failure. The new leader will reprocess this notification event");
+    }
+  }
+  private void dropSentryTablePrivileges(String dbName, String tableName) throws SentryNoSuchObjectException,
+      SentryInvalidInputException {
+    TSentryAuthorizable authorizable = new TSentryAuthorizable(hiveInstance);
+    authorizable.setDb(dbName);
+    authorizable.setTable(tableName);
+    try {
+      sentryStore.dropPrivilege(authorizable);
+    } catch (SentryStandbyException e) {
+      LOGGER.error("Seems like this process became a standby. " +
+          "Ignoring the failure. The new leader will reprocess this notification event");
+    }
+  }
+  private void renamePrivileges(String oldDbName, String oldTableName, String newDbName, String newTableName) throws
+      SentryNoSuchObjectException, SentryInvalidInputException, SentryStandbyException{
+    TSentryAuthorizable oldAuthorizable = new TSentryAuthorizable(hiveInstance);
+    oldAuthorizable.setDb(oldDbName);
+    oldAuthorizable.setTable(oldTableName);
+    TSentryAuthorizable newAuthorizable = new TSentryAuthorizable(hiveInstance);
+    newAuthorizable.setDb(newDbName);
+    newAuthorizable.setTable(newTableName);
+    try {
+      sentryStore.renamePrivilege(oldAuthorizable, newAuthorizable);
+    } catch (SentryStandbyException e) {
+      LOGGER.error("Seems like this process became a standby. " +
+          "Ignoring the failure. The new leader will reprocess this notification event");
+    }
+  }
+}


Mime
View raw message