eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qingwz...@apache.org
Subject incubator-eagle git commit: EAGLE-212 Fix AlertDataSourceEntity Bug in Hive web
Date Tue, 29 Mar 2016 08:53:08 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master c2c187789 -> a3986fccb


EAGLE-212 Fix AlertDataSourceEntity Bug in Hive web

https://issues.apache.org/jira/browse/EAGLE-212

Author: Zhao, Qingwen <qingwzhao@ebay.com>
Reviewer: Jilin, Jiang
Closed: #134


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a3986fcc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a3986fcc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a3986fcc

Branch: refs/heads/master
Commit: a3986fccbcb9e50d51184810f928441f6f24bc44
Parents: c2c1877
Author: Zhao, Qingwen <qingwzhao@ebay.com>
Authored: Tue Mar 29 16:50:25 2016 +0800
Committer: Zhao, Qingwen <qingwzhao@ebay.com>
Committed: Tue Mar 29 16:50:25 2016 +0800

----------------------------------------------------------------------
 .../src/main/bin/eagle-topology-init.sh         |  6 +-
 .../common/config/EagleConfigConstants.java     |  3 +
 .../resolver/MetadataAccessConfigRepo.java      | 25 ++++----
 .../audit/TestMetaDataAccessConfigRepo.java     | 55 +++++++++++++++++
 .../hbase/HbaseMetadataBrowseWebResource.java   | 16 +++--
 .../hbase/resolver/HbaseMetadataResolver.java   |  6 +-
 .../hdfs/resolver/HDFSResourceResolver.java     |  6 +-
 .../hdfs/rest/HDFSResourceWebResource.java      |  6 +-
 .../hive/dao/HiveMetadataAccessConfig.java      | 32 ++++++++++
 .../hive/dao/HiveMetadataAccessConfigDAO.java   |  3 +-
 .../dao/HiveMetadataAccessConfigDAOImpl.java    | 62 ++------------------
 .../hive/TestHiveDataSourceConfigDAOImpl.java   | 32 ----------
 12 files changed, 137 insertions(+), 115 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-assembly/src/main/bin/eagle-topology-init.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/eagle-topology-init.sh b/eagle-assembly/src/main/bin/eagle-topology-init.sh
index fc476b4..2d638bf 100755
--- a/eagle-assembly/src/main/bin/eagle-topology-init.sh
+++ b/eagle-assembly/src/main/bin/eagle-topology-init.sh
@@ -33,11 +33,11 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a
 
 echo ""
 echo "Importing applications for sample site ..."
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"
-d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hdfsAuditLog"},
"enabled": true, "config" : "{\"fs.defaultFS\":\"hdfs://sandbox.hortonworks.com:8020\"}"}]'
+curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"
-d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hdfsAuditLog"},
"enabled": true, "config" : "web.fs.defaultFS: \"hdfs://sandbox.hortonworks.com:8020\""}]'
 
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"
-d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hbaseSecurityLog"},
"enabled": true, "config" : "{\"hbase.zookeeper.property.clientPort\":\"2181\", \"hbase.zookeeper.quorum\":\"localhost\"}"}]'
+curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"
-d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hbaseSecurityLog"},
"enabled": true, "config" : "web.hbase.zookeeper.property.clientPort: \"2181\", web.hbase.zookeeper.quorum:
\"localhost\""}]'
 
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"
-d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hiveQueryLog"},
"enabled": true, "config":"{\"accessType\":\"metastoredb_jdbc\",\"password\":\"hive\",\"user\":\"hive\",\"jdbcDriverClassName\":\"com.mysql.jdbc.Driver\",\"jdbcUrl\":\"jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true\"}"}]'
+curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=SiteApplicationService"
-d '[{"prefix":"eagleSiteApplication","tags":{"site" : "sandbox", "application":"hiveQueryLog"},
"enabled": true, "config":"web.accessType:\"metastoredb_jdbc\",web.password:\"hive\",web.user:\"hive\",web.jdbcDriverClassName:\"com.mysql.jdbc.Driver\",web.jdbcUrl:\"jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true\""}]'
 
 echo ""
 echo "Importing application definitions ..."

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
b/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
index 720dc35..0a035b8 100644
--- a/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
+++ b/eagle-core/eagle-query/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigConstants.java
@@ -56,4 +56,7 @@ public final class EagleConfigConstants {
     public final static String DATA_SOURCE = "dataSource";
     public final static String APPLICATION = "application";
 
+    public final static String WEB_CONFIG = "web";
+    public final static String APP_CONFIG = "app";
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java
b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java
index f3fce8d..fa178c9 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/resolver/MetadataAccessConfigRepo.java
@@ -19,8 +19,11 @@
 package org.apache.eagle.security.resolver;
 
 
-import com.fasterxml.jackson.databind.ObjectMapper;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigValue;
 import org.apache.eagle.alert.entity.SiteApplicationServiceEntity;
+import org.apache.eagle.common.config.EagleConfigConstants;
 import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
 import org.apache.eagle.policy.common.Constants;
 import org.apache.eagle.service.generic.GenericEntityServiceResource;
@@ -35,24 +38,26 @@ import java.util.Map;
 public class MetadataAccessConfigRepo {
     private static Logger LOG = LoggerFactory.getLogger(MetadataAccessConfigRepo.class);
 
-    public Configuration getConfig(String application, String siteId) throws Exception {
+    public Config getConfig(String application, String siteId) throws Exception {
 
         GenericEntityServiceResource resource = new GenericEntityServiceResource();
         String queryFormat = Constants.SITE_APPLICATION_SERVICE_ENDPOINT_NAME + "[@application=\"%s\"
AND @site=\"%s\"]{*}";
-        GenericServiceAPIResponseEntity ret = resource.search(String.format(queryFormat,
application, siteId), null, null,Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0, null,
false);
+        GenericServiceAPIResponseEntity ret = resource.search(String.format(queryFormat,
application, siteId), null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false, 0,
null, false);
         List<SiteApplicationServiceEntity> list = (List<SiteApplicationServiceEntity>)
ret.getObj();
         if (list == null || list.size() == 0)
             throw new Exception("Config is empty for site=" + siteId +" application=" + application
+ ".");
-
-        ObjectMapper mapper = new ObjectMapper();
-        Map<String, String> configMap = mapper.readValue(list.get(0).getConfig(), Map.class);
-        return convert(configMap);
+        String originalConfigStr = list.get(0).getConfig();
+        Config originalConfig = ConfigFactory.parseString(originalConfigStr);
+        if(!originalConfig.hasPath(EagleConfigConstants.WEB_CONFIG)) {
+            throw new Exception("Fail to get WEB_CONFIG configurations for data classification");
+        }
+        return originalConfig.getConfig(EagleConfigConstants.WEB_CONFIG);
     }
 
-    private Configuration convert(Map<String, String> configMap) throws Exception {
+    public Configuration convert(Config originalConfig) throws Exception {
         Configuration config = new Configuration();
-        for (Map.Entry<String, String> entry : configMap.entrySet()) {
-            config.set(entry.getKey(), entry.getValue());
+        for (Map.Entry<String, ConfigValue> entry : originalConfig.entrySet()) {
+            config.set(entry.getKey().toString(), entry.getValue().unwrapped().toString());
         }
         return config;
     }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
new file mode 100644
index 0000000..146e757
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.crawler.audit;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigValue;
+import junit.framework.Assert;
+import org.apache.eagle.common.config.EagleConfigConstants;
+import org.junit.Test;
+
+import java.util.Map;
+
+public class TestMetaDataAccessConfigRepo {
+
+    @Test
+    public void testStringToConfig() {
+        String hdfsConfigStr = "web.fs.defaultFS: \"hdfs://sandbox.hortonworks.com:8020\"";
+        Config config = ConfigFactory.parseString(hdfsConfigStr);
+        Assert.assertTrue(config.hasPath(EagleConfigConstants.WEB_CONFIG));
+
+        String hiveConfigStr = "web.accessType:\"metastoredb_jdbc\",web.password:\"hive\",web.user:\"hive\",web.jdbcDriverClassName:\"com.mysql.jdbc.Driver\",web.jdbcUrl:\"jdbc:mysql://sandbox.hortonworks.com/hive?createDatabaseIfNotExist=true\"";
+        config = ConfigFactory.parseString(hiveConfigStr);
+        Config hiveConfig = null;
+        if(config.hasPath(EagleConfigConstants.WEB_CONFIG)) {
+            hiveConfig = config.getConfig(EagleConfigConstants.WEB_CONFIG);
+            Assert.assertTrue(hiveConfig.getString("accessType").equals("metastoredb_jdbc"));
+        }
+
+        String hbaseConfigStr = "web.hbase.zookeeper.property.clientPort: \"2181\", web.hbase.zookeeper.quorum:
\"localhost\"";
+        config = ConfigFactory.parseString(hbaseConfigStr);
+        Config hbaseConfig = null;
+        if(config.hasPath(EagleConfigConstants.WEB_CONFIG)) {
+            hbaseConfig = config.getConfig(EagleConfigConstants.WEB_CONFIG);
+            Assert.assertTrue(hbaseConfig.getString("hbase.zookeeper.property.clientPort").equals("2181"));
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java
b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java
index b15b15b..2daaa17 100644
--- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java
+++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java
@@ -16,6 +16,7 @@
  */
 package org.apache.eagle.service.security.hbase;
 
+import com.typesafe.config.Config;
 import org.apache.eagle.security.entity.HbaseResourceEntity;
 import org.apache.eagle.security.resolver.MetadataAccessConfigRepo;
 import org.apache.eagle.service.common.EagleExceptionWrapper;
@@ -46,8 +47,9 @@ public class HbaseMetadataBrowseWebResource {
         List<HbaseResourceEntity> values = new ArrayList<>();
         HbaseMetadataBrowseWebResponse response = new HbaseMetadataBrowseWebResponse();
         try {
-            Configuration config = repo.getConfig(HBASE_APPLICATION, site);
-            HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(config);
+            Config config = repo.getConfig(HBASE_APPLICATION, site);
+            Configuration conf = repo.convert(config);
+            HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf);
             namespaces = dao.getNamespaces();
 
         } catch (Exception e) {
@@ -74,8 +76,9 @@ public class HbaseMetadataBrowseWebResource {
         List<String> tables = null;
         List<HbaseResourceEntity> values = new ArrayList<>();
         try {
-            Configuration config = repo.getConfig(HBASE_APPLICATION, site);
-            HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(config);
+            Config config = repo.getConfig(HBASE_APPLICATION, site);
+            Configuration conf = repo.convert(config);
+            HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf);
             tables = dao.getTables(namespace);
         }catch(Exception ex){
             LOG.error("fail getting databases", ex);
@@ -104,8 +107,9 @@ public class HbaseMetadataBrowseWebResource {
         List<String> columns = null;
         List<HbaseResourceEntity> values = new ArrayList<>();
         try {
-            Configuration config = repo.getConfig(HBASE_APPLICATION, site);
-            HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(config);
+            Config config = repo.getConfig(HBASE_APPLICATION, site);
+            Configuration conf = repo.convert(config);
+            HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf);
             String tableName = String.format("%s:%s", namespace, table);
             columns = dao.getColumnFamilies(tableName);
         }catch(Exception ex){

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java
b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java
index c7b1b5c..7b13926 100644
--- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java
+++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java
@@ -19,6 +19,7 @@
 package org.apache.eagle.service.security.hbase.resolver;
 
 
+import com.typesafe.config.Config;
 import org.apache.eagle.security.resolver.MetadataAccessConfigRepo;
 import org.apache.eagle.service.alert.resolver.AttributeResolvable;
 import org.apache.eagle.service.alert.resolver.AttributeResolveException;
@@ -43,8 +44,9 @@ public class HbaseMetadataResolver implements AttributeResolvable<GenericAttribu
 
         try {
             MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo();
-            Configuration config = repo.getConfig(HbaseMetadataBrowseWebResource.HBASE_APPLICATION,
site);
-            HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(config);
+            Config config = repo.getConfig(HbaseMetadataBrowseWebResource.HBASE_APPLICATION,
site);
+            Configuration conf = repo.convert(config);
+            HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf);
 
             switch (subResources.length) {
                 case 1:

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java
b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java
index 71b67ec..4326c93 100644
--- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java
+++ b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/resolver/HDFSResourceResolver.java
@@ -21,6 +21,7 @@ import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import com.typesafe.config.Config;
 import org.apache.eagle.security.resolver.MetadataAccessConfigRepo;
 import org.apache.eagle.service.alert.resolver.AttributeResolvable;
 import org.apache.eagle.service.alert.resolver.AttributeResolveException;
@@ -55,8 +56,9 @@ public class HDFSResourceResolver  implements AttributeResolvable<GenericAttribu
 		List<String> result = new ArrayList<>();
 		MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo();
 		try {
-			Configuration config = repo.getConfig(HDFSResourceConstants.HDFS_APPLICATION, request.getSite().trim());
-			HDFSFileSystem fileSystem = new HDFSFileSystem(config);
+			Config config = repo.getConfig(HDFSResourceConstants.HDFS_APPLICATION, request.getSite().trim());
+			Configuration conf = repo.convert(config);
+			HDFSFileSystem fileSystem = new HDFSFileSystem(conf);
 			String query = request.getQuery().trim();
 			List<FileStatus> fileStatuses = null;
 			if(query.endsWith("/")) {

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java
b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java
index 49e4372..083c9f8 100644
--- a/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java
+++ b/eagle-security/eagle-security-hdfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/HDFSResourceWebResource.java
@@ -26,6 +26,7 @@ import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.MediaType;
 
+import com.typesafe.config.Config;
 import org.apache.eagle.security.entity.FileStatusEntity;
 import org.apache.eagle.security.resolver.MetadataAccessConfigRepo;
 import org.apache.eagle.service.common.EagleExceptionWrapper;
@@ -60,8 +61,9 @@ public class HDFSResourceWebResource
 		List<FileStatus> fileStatuses = null;
 		try {
 			validator.validate(site, filePath); // First Step would be validating Request
-			Configuration config = repo.getConfig(HDFSResourceConstants.HDFS_APPLICATION, site);
-			HDFSFileSystem fileSystem = new HDFSFileSystem(config);
+			Config config = repo.getConfig(HDFSResourceConstants.HDFS_APPLICATION, site);
+			Configuration conf = repo.convert(config);
+			HDFSFileSystem fileSystem = new HDFSFileSystem(conf);
 			fileStatuses = fileSystem.browse(filePath);
 			// Join with File Sensitivity Info
 			HDFSResourceSensitivityDataJoiner joiner = new HDFSResourceSensitivityDataJoiner();

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfig.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfig.java
b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfig.java
index 92f9707..3345436 100644
--- a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfig.java
+++ b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfig.java
@@ -16,6 +16,8 @@
  */
 package org.apache.eagle.service.security.hive.dao;
 
+import com.typesafe.config.Config;
+
 public class HiveMetadataAccessConfig {
     private String accessType;
     private String user;
@@ -67,4 +69,34 @@ public class HiveMetadataAccessConfig {
     public String toString(){
         return "user:" + user +",jdbcDriverClassName:" + jdbcDriverClassName + ",jdbcUrl:"
+ jdbcUrl;
     }
+
+    public final static class HIVECONF {
+        public final static String ACCESSTYPE = "accessType";
+        public final static String USER = "user";
+        public final static String PASSWD = "password";
+        public final static String JDBCDRIVER = "jdbcDriverClassName";
+        public final static String JDBCURL = "jdbcUrl";
+    }
+
+    public static HiveMetadataAccessConfig config2Entity(Config config) {
+        HiveMetadataAccessConfig hiveConf = new HiveMetadataAccessConfig();
+        if(config.hasPath(HIVECONF.ACCESSTYPE)) {
+            hiveConf.setAccessType(config.getString(HIVECONF.ACCESSTYPE));
+        }
+        if(config.hasPath(HIVECONF.USER)) {
+            hiveConf.setUser(config.getString(HIVECONF.USER));
+        }
+        if(config.hasPath(HIVECONF.PASSWD)) {
+            hiveConf.setPassword(config.getString(HIVECONF.PASSWD));
+        }
+        if(config.hasPath(HIVECONF.JDBCDRIVER)) {
+            hiveConf.setJdbcDriverClassName(config.getString(HIVECONF.JDBCDRIVER));
+        }
+        if(config.hasPath(HIVECONF.JDBCURL)) {
+            hiveConf.setJdbcUrl(config.getString(HIVECONF.JDBCURL));
+        }
+        return hiveConf;
+    }
 }
+
+

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAO.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAO.java
b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAO.java
index bd7e64c..414fa60 100644
--- a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAO.java
+++ b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAO.java
@@ -19,8 +19,7 @@ package org.apache.eagle.service.security.hive.dao;
 import java.util.Map;
 
 public interface HiveMetadataAccessConfigDAO {
-    // site to HiveMetadataAccessConfig
-    Map<String, HiveMetadataAccessConfig> getAllConfigs() throws Exception;
+
     // HiveMetadataAccessConfig for one site
     HiveMetadataAccessConfig getConfig(String site) throws Exception;
 }

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAOImpl.java
b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAOImpl.java
index b4d3f9e..f4f41e0 100644
--- a/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAOImpl.java
+++ b/eagle-security/eagle-security-hive-web/src/main/java/org/apache/eagle/service/security/hive/dao/HiveMetadataAccessConfigDAOImpl.java
@@ -16,9 +16,12 @@
  */
 package org.apache.eagle.service.security.hive.dao;
 
+import com.typesafe.config.Config;
 import org.apache.eagle.alert.entity.AlertDataSourceEntity;
 import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
 import org.apache.eagle.log.entity.ListQueryAPIResponseEntity;
+import org.apache.eagle.policy.common.Constants;
+import org.apache.eagle.security.resolver.MetadataAccessConfigRepo;
 import org.apache.eagle.service.generic.GenericEntityServiceResource;
 import org.apache.eagle.service.generic.ListQueryResource;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -32,64 +35,11 @@ import java.util.Map;
 public class HiveMetadataAccessConfigDAOImpl implements HiveMetadataAccessConfigDAO {
     private static Logger LOG = LoggerFactory.getLogger(HiveMetadataAccessConfigDAOImpl.class);
 
-    // site to HiveMetadataAccessConfig
-//    @Override
-//    public Map<String, HiveMetadataAccessConfig> getAllConfigs() throws Exception{
-//        ListQueryResource resource = new ListQueryResource();
-//        /* parameters are: query, startTime, endTime, pageSzie, startRowkey, treeAgg, timeSeries,
intervalmin, top, filterIfMissing,
-//        * parallel, metricName*/
-//        ListQueryAPIResponseEntity ret = resource.listQuery("AlertDataSourceService[@dataSource=\"hiveQueryLog\"]{*}",
null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false,
-//                0, null);
-//        List<AlertDataSourceEntity> list = (List<AlertDataSourceEntity>) ret.getObj();
-//        Map<String, HiveMetadataAccessConfig> siteHiveConfigs = new HashMap<String,
HiveMetadataAccessConfig>();
-//        for(AlertDataSourceEntity hiveDataSource : list){
-//            siteHiveConfigs.put(hiveDataSource.getTags().get("site"), convert(hiveDataSource.getConfig()));
-//        }
-//        return siteHiveConfigs;
-//    }
-
-    // site to HiveMetadataAccessConfig
-    @Override
-    public Map<String, HiveMetadataAccessConfig> getAllConfigs() throws Exception{
-        GenericEntityServiceResource resource = new GenericEntityServiceResource();
-
-        /* parameters are: query, startTime, endTime, pageSzie, startRowkey, treeAgg, timeSeries,
intervalmin, top, filterIfMissing,
-        * parallel, metricName*/
-        GenericServiceAPIResponseEntity ret = resource.search("AlertDataSourceService[@dataSource=\"hiveQueryLog\"]{*}",
null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false,
-                0, null, false);
-        List<AlertDataSourceEntity> list = (List<AlertDataSourceEntity>) ret.getObj();
-        Map<String, HiveMetadataAccessConfig> siteHiveConfigs = new HashMap<String,
HiveMetadataAccessConfig>();
-        for(AlertDataSourceEntity hiveDataSource : list){
-            siteHiveConfigs.put(hiveDataSource.getTags().get("site"), convert(hiveDataSource.getConfig()));
-        }
-        return siteHiveConfigs;
-    }
-
-
-    private HiveMetadataAccessConfig convert(String config){
-        ObjectMapper mapper = new ObjectMapper();
-        HiveMetadataAccessConfig c;
-        try {
-            c = mapper.readValue(config, HiveMetadataAccessConfig.class);
-        }catch(Exception ex){
-            LOG.error("config block could be broken", ex);
-            throw new BadHiveMetadataAccessConfigException(ex);
-        }
-        return c;
-    }
-
     // HiveMetadataAccessConfig for one site
     @Override
     public HiveMetadataAccessConfig getConfig(String site) throws Exception{
-        GenericEntityServiceResource resource = new GenericEntityServiceResource();
-        /* parameters are: query, startTime, endTime, pageSzie, startRowkey, treeAgg, timeSeries,
intervalmin, top, filterIfMissing,
-        * parallel, metricName*/
-        String queryFormat = "AlertDataSourceService[@dataSource=\"hiveQueryLog\" AND @site=\"%s\"]{*}";
-        GenericServiceAPIResponseEntity ret = resource.search(String.format(queryFormat,
site), null, null, Integer.MAX_VALUE, null, false, false, 0L, 0, false,
-                0, null, false);
-        List<AlertDataSourceEntity> list = (List<AlertDataSourceEntity>) ret.getObj();
-        if(list == null || list.size() ==0)
-            throw new BadHiveMetadataAccessConfigException("config is empty for site " +
site);
-        return convert(list.get(0).getConfig());
+        MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo();
+        Config config = repo.getConfig("hiveQueryLog", site);
+        return HiveMetadataAccessConfig.config2Entity(config);
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a3986fcc/eagle-security/eagle-security-hive-web/src/test/java/org/apache/eagle/service/security/hive/TestHiveDataSourceConfigDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive-web/src/test/java/org/apache/eagle/service/security/hive/TestHiveDataSourceConfigDAOImpl.java
b/eagle-security/eagle-security-hive-web/src/test/java/org/apache/eagle/service/security/hive/TestHiveDataSourceConfigDAOImpl.java
deleted file mode 100644
index 541548f..0000000
--- a/eagle-security/eagle-security-hive-web/src/test/java/org/apache/eagle/service/security/hive/TestHiveDataSourceConfigDAOImpl.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.apache.eagle.service.security.hive;/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import org.apache.eagle.service.security.hive.dao.HiveMetadataAccessConfigDAOImpl;
-
-import org.junit.Test;
-
-public class TestHiveDataSourceConfigDAOImpl {
-    //@Test
-    public void testDataSourceConfigLoader() throws Exception{
-        HiveMetadataAccessConfigDAOImpl daoImpl = new HiveMetadataAccessConfigDAOImpl();
-        System.out.println(daoImpl.getAllConfigs());
-    }
-
-    @Test
-    public void test() {
-
-    }
-}


Mime
View raw message