Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 416FE200C5A for ; Tue, 18 Apr 2017 10:05:36 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 3FE40160B90; Tue, 18 Apr 2017 08:05:36 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 16B31160BA1 for ; Tue, 18 Apr 2017 10:05:33 +0200 (CEST) Received: (qmail 15706 invoked by uid 500); 18 Apr 2017 08:05:32 -0000 Mailing-List: contact commits-help@eagle.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@eagle.apache.org Delivered-To: mailing list commits@eagle.apache.org Received: (qmail 15658 invoked by uid 99); 18 Apr 2017 08:05:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 18 Apr 2017 08:05:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8CD17DFDAC; Tue, 18 Apr 2017 08:05:32 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jaysen@apache.org To: commits@eagle.apache.org Date: Tue, 18 Apr 2017 08:05:33 -0000 Message-Id: <0ca3ec51bfa847efa9f50e1f1befb853@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/3] eagle git commit: HBase Naming that unify `Hbase` and `HBase` into `HBase` archived-at: Tue, 18 Apr 2017 08:05:36 -0000 http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java deleted file mode 100644 index 467cc2f..0000000 --- a/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.eagle.security.hbase; - -import org.junit.Assert; -import org.junit.Test; - - -public class TestHbaseAuditLogParser { - HbaseAuditLogParser parser = new HbaseAuditLogParser(); - - @Test - public void test() throws Exception { - String log = "2015-08-11 13:31:03,729 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Table permission granted; remote address: /127.0.0.1; request: get; context: (user=eagle,scope=hbase:namespace,family=info, action=READ)"; - HbaseAuditLogObject obj = parser.parse(log); - Assert.assertEquals(obj.action, "READ"); - Assert.assertEquals(obj.host, "127.0.0.1"); - Assert.assertEquals(obj.scope, "hbase:namespace:info"); - } - - @Test - public void test2() throws Exception { - String log = "2015-08-04 12:29:03,073 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Global check allowed; remote address: ; request: preOpen; context: (user=eagle, scope=GLOBAL, family=, action=ADMIN)"; - HbaseAuditLogObject obj = parser.parse(log); - Assert.assertEquals(obj.action, "ADMIN"); - Assert.assertEquals(obj.host, ""); - Assert.assertEquals(obj.scope, "GLOBAL"); - } -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResource.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResource.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResource.java new file mode 100644 index 0000000..91dd5a3 --- /dev/null +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResource.java @@ -0,0 +1,189 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. See the NOTICE file distributed with + * * this work for additional information regarding copyright ownership. + * * The ASF licenses this file to You under the Apache License, Version 2.0 + * * (the "License"); you may not use this file except in compliance with + * * the License. You may obtain a copy of the License at + * *

+ * * http://www.apache.org/licenses/LICENSE-2.0 + * *

+ * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ +package org.apache.eagle.service.security.hbase; + +import com.google.inject.Inject; +import com.typesafe.config.Config; +import org.apache.eagle.metadata.model.ApplicationEntity; +import org.apache.eagle.metadata.service.ApplicationEntityService; +import org.apache.eagle.security.entity.HBaseResourceEntity; +import org.apache.eagle.security.service.HBaseSensitivityEntity; +import org.apache.eagle.security.service.ISecurityMetadataDAO; +import org.apache.eagle.security.service.MetadataDaoFactory; +import org.apache.eagle.service.common.EagleExceptionWrapper; +import org.apache.eagle.service.security.hbase.dao.HBaseMetadataDAOImpl; +import org.apache.hadoop.conf.Configuration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import java.util.*; +import java.util.regex.Pattern; + +@Path("/hbaseResource") +public class HBaseMetadataBrowseWebResource { + private static Logger LOG = LoggerFactory.getLogger(HBaseMetadataBrowseWebResource.class); + final public static String HBASE_APPLICATION = "HBaseAuditLogApplication"; + + private ApplicationEntityService entityService; + private ISecurityMetadataDAO dao; + + @Inject + public HBaseMetadataBrowseWebResource(ApplicationEntityService entityService, Config eagleServerConfig){ + this.entityService = entityService; + this.dao = MetadataDaoFactory.getMetadataDAO(eagleServerConfig); + } + + private Map> getAllSensitivities(){ + Map> all = new HashMap<>(); + Collection entities = dao.listHBaseSensitivities(); + for(HBaseSensitivityEntity entity : entities){ + if(!all.containsKey(entity.getSite())){ + all.put(entity.getSite(), new HashMap<>()); + } + all.get(entity.getSite()).put(entity.getHBaseResource(), entity.getSensitivityType()); + } + return all; + } + + private Map getAppConfig(String site, String appType){ + ApplicationEntity entity = entityService.getBySiteIdAndAppType(site, appType); + return entity.getConfiguration(); + } + + + private Configuration convert(Map originalConfig) throws Exception { + Configuration config = new Configuration(); + for (Map.Entry entry : originalConfig.entrySet()) { + config.set(entry.getKey(), entry.getValue().toString()); + } + return config; + } + + @Path("/namespaces") + @GET + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public HBaseMetadataBrowseWebResponse getNamespace(@QueryParam("site") String site) { + List namespaces = null; + List values = new ArrayList<>(); + HBaseMetadataBrowseWebResponse response = new HBaseMetadataBrowseWebResponse(); + try { + Map config = getAppConfig(site, HBASE_APPLICATION); + Configuration conf = convert(config); + HBaseMetadataDAOImpl dao = new HBaseMetadataDAOImpl(conf); + namespaces = dao.getNamespaces(); + + } catch (Exception e) { + e.printStackTrace(); + } + if(namespaces != null) { + for (String ns : namespaces) { + Set childSensitiveTypes = new HashSet<>(); + String senstiveType = checkSensitivity(site, ns, childSensitiveTypes); + values.add(new HBaseResourceEntity(ns, ns, null, null, senstiveType, childSensitiveTypes)); + } + } + response.setObj(values); + return response; + } + + @Path("/tables") + @GET + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public HBaseMetadataBrowseWebResponse getTables(@QueryParam("site") String site, @QueryParam("namespace") String namespace){ + // delegate to HiveMetadataDAO + HBaseMetadataBrowseWebResponse response = new HBaseMetadataBrowseWebResponse(); + List tables = null; + List values = new ArrayList<>(); + try { + Map config = getAppConfig(site, HBASE_APPLICATION); + Configuration conf = convert(config); + HBaseMetadataDAOImpl dao = new HBaseMetadataDAOImpl(conf); + tables = dao.getTables(namespace); + }catch(Exception ex){ + LOG.error("fail getting databases", ex); + response.setException(EagleExceptionWrapper.wrap(ex)); + } + if(tables != null) { + for (String table : tables) { + String resource = String.format("%s:%s", namespace, table); + Set childSensitiveTypes = new HashSet<>(); + String sensitiveType = checkSensitivity(site, resource, childSensitiveTypes); + values.add(new HBaseResourceEntity(resource, namespace, table, null, sensitiveType, childSensitiveTypes)); + } + } + response.setObj(values); + return response; + } + + @Path("/columns") + @GET + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public HBaseMetadataBrowseWebResponse getColumns(@QueryParam("site") String site, @QueryParam("namespace") String namespace, @QueryParam("table") String table){ + // delegate to HiveMetadataDAO + HBaseMetadataBrowseWebResponse response = new HBaseMetadataBrowseWebResponse(); + List columns = null; + List values = new ArrayList<>(); + try { + Map config = getAppConfig(site, HBASE_APPLICATION); + Configuration conf = convert(config); + HBaseMetadataDAOImpl dao = new HBaseMetadataDAOImpl(conf); + String tableName = String.format("%s:%s", namespace, table); + columns = dao.getColumnFamilies(tableName); + }catch(Exception ex){ + LOG.error("fail getting databases", ex); + response.setException(EagleExceptionWrapper.wrap(ex)); + } + if(columns != null) { + for (String col : columns) { + String resource = String.format("%s:%s:%s", namespace, table, col); + Set childSensitiveTypes = new HashSet<>(); + String sensitiveType = checkSensitivity(site, resource, childSensitiveTypes); + values.add(new HBaseResourceEntity(resource, namespace, table, col, sensitiveType, childSensitiveTypes)); + } + } + response.setObj(values); + return response; + } + + + + String checkSensitivity(String site, String resource, Set childSensitiveTypes) { + Map> maps = getAllSensitivities(); + String sensitiveType = null; + if (maps != null && maps.get(site) != null) { + Map map = maps.get(site); + for (String r : map.keySet()) { + Pattern pattern = Pattern.compile("^" + resource); + boolean isMatched = Pattern.matches(r, resource); + if (isMatched) { + sensitiveType = map.get(r); + } + else if (pattern.matcher(r).find()){ + childSensitiveTypes.add(map.get(r)); + } + } + } + return sensitiveType; + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResponse.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResponse.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResponse.java new file mode 100644 index 0000000..929989f --- /dev/null +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResponse.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.service.security.hbase; + + +import org.apache.eagle.security.entity.HBaseResourceEntity; + +import java.util.List; + +public class HBaseMetadataBrowseWebResponse { + private String exception; + private List obj; + + public List getObj() { + return obj; + } + + public void setObj(List obj) { + this.obj = obj; + } + + public String getException() { + return exception; + } + + public void setException(String exception) { + this.exception = exception; + } + +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java deleted file mode 100644 index 69be2e5..0000000 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * - * * Licensed to the Apache Software Foundation (ASF) under one or more - * * contributor license agreements. See the NOTICE file distributed with - * * this work for additional information regarding copyright ownership. - * * The ASF licenses this file to You under the Apache License, Version 2.0 - * * (the "License"); you may not use this file except in compliance with - * * the License. You may obtain a copy of the License at - * *

- * * http://www.apache.org/licenses/LICENSE-2.0 - * *

- * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ -package org.apache.eagle.service.security.hbase; - -import com.google.inject.Inject; -import com.typesafe.config.Config; -import org.apache.eagle.metadata.model.ApplicationEntity; -import org.apache.eagle.metadata.service.ApplicationEntityService; -import org.apache.eagle.security.entity.HbaseResourceEntity; -import org.apache.eagle.security.resolver.MetadataAccessConfigRepo; -import org.apache.eagle.security.service.HBaseSensitivityEntity; -import org.apache.eagle.security.service.ISecurityMetadataDAO; -import org.apache.eagle.security.service.MetadataDaoFactory; -import org.apache.eagle.service.common.EagleExceptionWrapper; -import org.apache.eagle.service.security.hbase.dao.HbaseMetadataDAOImpl; -import org.apache.hadoop.conf.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import java.util.*; -import java.util.regex.Pattern; - -@Path("/hbaseResource") -public class HbaseMetadataBrowseWebResource { - private static Logger LOG = LoggerFactory.getLogger(HbaseMetadataBrowseWebResource.class); - final public static String HBASE_APPLICATION = "HBaseAuditLogApplication"; - - private ApplicationEntityService entityService; - private ISecurityMetadataDAO dao; - - @Inject - public HbaseMetadataBrowseWebResource(ApplicationEntityService entityService, Config eagleServerConfig){ - this.entityService = entityService; - this.dao = MetadataDaoFactory.getMetadataDAO(eagleServerConfig); - } - - private Map> getAllSensitivities(){ - Map> all = new HashMap<>(); - Collection entities = dao.listHBaseSensitivities(); - for(HBaseSensitivityEntity entity : entities){ - if(!all.containsKey(entity.getSite())){ - all.put(entity.getSite(), new HashMap<>()); - } - all.get(entity.getSite()).put(entity.getHbaseResource(), entity.getSensitivityType()); - } - return all; - } - - private Map getAppConfig(String site, String appType){ - ApplicationEntity entity = entityService.getBySiteIdAndAppType(site, appType); - return entity.getConfiguration(); - } - - - private Configuration convert(Map originalConfig) throws Exception { - Configuration config = new Configuration(); - for (Map.Entry entry : originalConfig.entrySet()) { - config.set(entry.getKey(), entry.getValue().toString()); - } - return config; - } - - @Path("/namespaces") - @GET - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) - public HbaseMetadataBrowseWebResponse getNamespace(@QueryParam("site") String site) { - List namespaces = null; - List values = new ArrayList<>(); - HbaseMetadataBrowseWebResponse response = new HbaseMetadataBrowseWebResponse(); - try { - Map config = getAppConfig(site, HBASE_APPLICATION); - Configuration conf = convert(config); - HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf); - namespaces = dao.getNamespaces(); - - } catch (Exception e) { - e.printStackTrace(); - } - if(namespaces != null) { - for (String ns : namespaces) { - Set childSensitiveTypes = new HashSet<>(); - String senstiveType = checkSensitivity(site, ns, childSensitiveTypes); - values.add(new HbaseResourceEntity(ns, ns, null, null, senstiveType, childSensitiveTypes)); - } - } - response.setObj(values); - return response; - } - - @Path("/tables") - @GET - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) - public HbaseMetadataBrowseWebResponse getTables(@QueryParam("site") String site, @QueryParam("namespace") String namespace){ - // delegate to HiveMetadataDAO - HbaseMetadataBrowseWebResponse response = new HbaseMetadataBrowseWebResponse(); - List tables = null; - List values = new ArrayList<>(); - try { - Map config = getAppConfig(site, HBASE_APPLICATION); - Configuration conf = convert(config); - HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf); - tables = dao.getTables(namespace); - }catch(Exception ex){ - LOG.error("fail getting databases", ex); - response.setException(EagleExceptionWrapper.wrap(ex)); - } - if(tables != null) { - for (String table : tables) { - String resource = String.format("%s:%s", namespace, table); - Set childSensitiveTypes = new HashSet<>(); - String sensitiveType = checkSensitivity(site, resource, childSensitiveTypes); - values.add(new HbaseResourceEntity(resource, namespace, table, null, sensitiveType, childSensitiveTypes)); - } - } - response.setObj(values); - return response; - } - - @Path("/columns") - @GET - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_JSON) - public HbaseMetadataBrowseWebResponse getColumns(@QueryParam("site") String site, @QueryParam("namespace") String namespace, @QueryParam("table") String table){ - // delegate to HiveMetadataDAO - HbaseMetadataBrowseWebResponse response = new HbaseMetadataBrowseWebResponse(); - List columns = null; - List values = new ArrayList<>(); - try { - Map config = getAppConfig(site, HBASE_APPLICATION); - Configuration conf = convert(config); - HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf); - String tableName = String.format("%s:%s", namespace, table); - columns = dao.getColumnFamilies(tableName); - }catch(Exception ex){ - LOG.error("fail getting databases", ex); - response.setException(EagleExceptionWrapper.wrap(ex)); - } - if(columns != null) { - for (String col : columns) { - String resource = String.format("%s:%s:%s", namespace, table, col); - Set childSensitiveTypes = new HashSet<>(); - String sensitiveType = checkSensitivity(site, resource, childSensitiveTypes); - values.add(new HbaseResourceEntity(resource, namespace, table, col, sensitiveType, childSensitiveTypes)); - } - } - response.setObj(values); - return response; - } - - - - String checkSensitivity(String site, String resource, Set childSensitiveTypes) { - Map> maps = getAllSensitivities(); - String sensitiveType = null; - if (maps != null && maps.get(site) != null) { - Map map = maps.get(site); - for (String r : map.keySet()) { - Pattern pattern = Pattern.compile("^" + resource); - boolean isMatched = Pattern.matches(r, resource); - if (isMatched) { - sensitiveType = map.get(r); - } - else if (pattern.matcher(r).find()){ - childSensitiveTypes.add(map.get(r)); - } - } - } - return sensitiveType; - } -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResponse.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResponse.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResponse.java deleted file mode 100644 index 960a411..0000000 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResponse.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.service.security.hbase; - - -import org.apache.eagle.security.entity.HbaseResourceEntity; - -import java.util.List; - -public class HbaseMetadataBrowseWebResponse { - private String exception; - private List obj; - - public List getObj() { - return obj; - } - - public void setObj(List obj) { - this.obj = obj; - } - - public String getException() { - return exception; - } - - public void setException(String exception) { - this.exception = exception; - } - -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HBaseMetadataDAOImpl.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HBaseMetadataDAOImpl.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HBaseMetadataDAOImpl.java new file mode 100644 index 0000000..b1a9c20 --- /dev/null +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HBaseMetadataDAOImpl.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.service.security.hbase.dao; + + +import org.apache.eagle.security.util.HadoopSecurityUtil; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class HBaseMetadataDAOImpl { + private final static Logger LOG = LoggerFactory.getLogger(HBaseMetadataDAOImpl.class); + + private Configuration hBaseConfiguration; + + public HBaseMetadataDAOImpl(Configuration config) { + this.hBaseConfiguration = HBaseConfiguration.create(); + this.hBaseConfiguration.addResource(config); + //this.hBaseConfiguration.set("hbase.zookeeper.quorum", this.config.getZkQuorum()); + //this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", this.config.getZkClientPort()); + } + + private HBaseAdmin getHBaseAdmin() throws IOException { + HadoopSecurityUtil.login(hBaseConfiguration); + return new HBaseAdmin(this.hBaseConfiguration); + } + + public List getNamespaces() throws IOException { + List ret = new ArrayList(); + HBaseAdmin admin = this.getHBaseAdmin(); + NamespaceDescriptor [] ns = admin.listNamespaceDescriptors(); + for(NamespaceDescriptor n : ns) { + ret.add(n.getName()); + } + closeHBaseConnection(admin); + return ret; + } + + public List getTables(String namespace) throws IOException { + List ret = new ArrayList(); + HBaseAdmin admin = this.getHBaseAdmin(); + TableName[] tables = admin.listTableNamesByNamespace(namespace); + + for(TableName tableName : tables) { + ret.add(tableName.getQualifierAsString()); + } + closeHBaseConnection(admin); + return ret; + } + + public List getColumnFamilies(String tableName) throws IOException { + List ret = new ArrayList(); + HBaseAdmin admin = this.getHBaseAdmin(); + HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName.getBytes()); + + HColumnDescriptor [] cfs = tableDescriptor.getColumnFamilies(); + for(HColumnDescriptor cf : cfs) { + ret.add(cf.getNameAsString()); + } + closeHBaseConnection(admin); + return ret; + } + + private void closeHBaseConnection(HBaseAdmin admin) { + if(admin != null) { + try { + admin.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java deleted file mode 100644 index ed52bd7..0000000 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.service.security.hbase.dao; - - -import org.apache.eagle.security.util.HadoopSecurityUtil; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class HbaseMetadataDAOImpl { - private final static Logger LOG = LoggerFactory.getLogger(HbaseMetadataDAOImpl.class); - - private Configuration hBaseConfiguration; - - public HbaseMetadataDAOImpl(Configuration config) { - this.hBaseConfiguration = HBaseConfiguration.create(); - this.hBaseConfiguration.addResource(config); - //this.hBaseConfiguration.set("hbase.zookeeper.quorum", this.config.getZkQuorum()); - //this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", this.config.getZkClientPort()); - } - - private HBaseAdmin getHBaseAdmin() throws IOException { - HadoopSecurityUtil.login(hBaseConfiguration); - return new HBaseAdmin(this.hBaseConfiguration); - } - - public List getNamespaces() throws IOException { - List ret = new ArrayList(); - HBaseAdmin admin = this.getHBaseAdmin(); - NamespaceDescriptor [] ns = admin.listNamespaceDescriptors(); - for(NamespaceDescriptor n : ns) { - ret.add(n.getName()); - } - closeHbaseConnection(admin); - return ret; - } - - public List getTables(String namespace) throws IOException { - List ret = new ArrayList(); - HBaseAdmin admin = this.getHBaseAdmin(); - TableName[] tables = admin.listTableNamesByNamespace(namespace); - - for(TableName tableName : tables) { - ret.add(tableName.getQualifierAsString()); - } - closeHbaseConnection(admin); - return ret; - } - - public List getColumnFamilies(String tableName) throws IOException { - List ret = new ArrayList(); - HBaseAdmin admin = this.getHBaseAdmin(); - HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName.getBytes()); - - HColumnDescriptor [] cfs = tableDescriptor.getColumnFamilies(); - for(HColumnDescriptor cf : cfs) { - ret.add(cf.getNameAsString()); - } - closeHbaseConnection(admin); - return ret; - } - - private void closeHbaseConnection(HBaseAdmin admin) { - if(admin != null) { - try { - admin.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseActionResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseActionResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseActionResolver.java new file mode 100644 index 0000000..2b34670 --- /dev/null +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseActionResolver.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.service.security.hbase.resolver; + + +import com.typesafe.config.Config; +import org.apache.eagle.metadata.service.ApplicationEntityService; +import org.apache.eagle.security.resolver.AbstractCommandResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + + +public class HBaseActionResolver extends AbstractCommandResolver { + public HBaseActionResolver(ApplicationEntityService entityService, Config eagleServerConfig){ + + } + private final static Logger LOG = LoggerFactory.getLogger(HBaseActionResolver.class); + private final String [] cmdStrs = {"ADMIN", "READ", "WRITE", "CREATE", "EXECUTE"}; + + @Override + public void init() { + this.setCommands(Arrays.asList(cmdStrs)); + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseMetadataResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseMetadataResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseMetadataResolver.java new file mode 100644 index 0000000..7405f25 --- /dev/null +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseMetadataResolver.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.service.security.hbase.resolver; + +import com.typesafe.config.Config; +import org.apache.eagle.metadata.model.ApplicationEntity; +import org.apache.eagle.metadata.service.ApplicationEntityService; +import org.apache.eagle.service.alert.resolver.AttributeResolvable; +import org.apache.eagle.service.alert.resolver.AttributeResolveException; +import org.apache.eagle.service.alert.resolver.BadAttributeResolveRequestException; +import org.apache.eagle.service.alert.resolver.GenericAttributeResolveRequest; +import org.apache.eagle.service.security.hbase.HBaseMetadataBrowseWebResource; +import org.apache.eagle.service.security.hbase.dao.HBaseMetadataDAOImpl; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + + +public class HBaseMetadataResolver implements AttributeResolvable { + private ApplicationEntityService entityService; + public HBaseMetadataResolver(ApplicationEntityService entityService, Config eagleServerConfig){ + this.entityService = entityService; + } + + private Map getAppConfig(String site, String appType){ + ApplicationEntity entity = entityService.getBySiteIdAndAppType(site, appType); + return entity.getConfiguration(); + } + + private Configuration convert(Map originalConfig) throws Exception { + Configuration config = new Configuration(); + for (Map.Entry entry : originalConfig.entrySet()) { + config.set(entry.getKey().toString(), entry.getValue().toString()); + } + return config; + } + + @Override + public List resolve(GenericAttributeResolveRequest request) throws AttributeResolveException { + String query = request.getQuery().trim(); + String site = request.getSite().trim(); + String[] subResources = query.split(":"); + + try { + Map config = getAppConfig(site, HBaseMetadataBrowseWebResource.HBASE_APPLICATION); + Configuration conf = convert(config); + HBaseMetadataDAOImpl dao = new HBaseMetadataDAOImpl(conf); + + switch (subResources.length) { + case 1: + if(query.endsWith(":")) + return filterAndCombineAttribue(query, dao.getTables(subResources[0]), null); + else + return filterAndCombineAttribue("", dao.getNamespaces(), subResources[0]); + case 2: + if(query.endsWith(":")) + return filterAndCombineAttribue(query, dao.getColumnFamilies(subResources[1]), null); + else + return filterAndCombineAttribue(subResources[0], dao.getTables(subResources[0]), subResources[1]); + case 3: + return filterAndCombineAttribue(String.format("%s:%s", subResources[0], subResources[1]), dao.getColumnFamilies(subResources[1]), subResources[2]); + } + + } catch (IOException e) { + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + } + + return null; + } + + @Override + public void validateRequest(GenericAttributeResolveRequest request) throws BadAttributeResolveRequestException { + String query = request.getQuery(); + String site = request.getSite(); + if (query == null || query.startsWith(":") || !Pattern.matches("[a-zA-Z:]+", query) || site == null || site.length() == 0) { + throw new BadAttributeResolveRequestException("hBase resource must be {\"site\":\"${site}\", \"query\"=\"{namespace}:{table}:{columnFamily}\""); + } + } + + public List filterAndCombineAttribue(String prefix, List attrs, String target) { + List result = new ArrayList<>(); + String format = (prefix == "" ? "%s%s" : "%s:%s"); + + if(target == null) { + for (String attr : attrs){ + result.add(String.format("%s%s", prefix, attr)); + } + } else { + Pattern pattern = Pattern.compile("^" + target, Pattern.CASE_INSENSITIVE); + for (String attr : attrs) { + if (pattern.matcher(attr).find()) { + result.add(String.format(format, prefix, attr)); + } + } + if (result.size() == 0) { + for (String attr : attrs) { + result.add(String.format(format, prefix, attr)); + } + } + } + return result; + } + + @Override + public Class getRequestClass() { + return GenericAttributeResolveRequest.class; + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseRequestResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseRequestResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseRequestResolver.java new file mode 100644 index 0000000..483ba82 --- /dev/null +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseRequestResolver.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.service.security.hbase.resolver; + + +import com.typesafe.config.Config; +import org.apache.eagle.metadata.service.ApplicationEntityService; +import org.apache.eagle.security.resolver.AbstractCommandResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + + +public class HBaseRequestResolver extends AbstractCommandResolver { + private final static Logger LOG = LoggerFactory.getLogger(HBaseRequestResolver.class); + + public HBaseRequestResolver(ApplicationEntityService entityService, Config eagleServerConfig){ + + } + + private final String [] master = {"createTable", "modifyTable", "deleteTable", "truncateTable", "addColumn", "modifyColumn", + "deleteColumn", "enableTable", "disableTable", "disableAclTable", "move", "assign", "unassign", + "regionOffline", "balance", "balanceSwitch", "shutdown", "stopMaster", "snapshot", + "listSnapshot", "cloneSnapshot", "restoreSnapshot", "deleteSnapshot", "createNamespace", + "deleteNamespace", "modifyNamespace", "getNamespaceDescriptor", "listNamespaceDescriptors*", + "flushTable", "getTableDescriptors*", "getTableNames*", "setUserQuota", "setTableQuota", + "setNamespaceQuota"}; + private final String [] region = {"openRegion", "closeRegion", "flush", "split", "compact", "getClosestRowBefore", "getOp", "exists", + "put", "delete", "batchMutate", "checkAndPut", "checkAndPutAfterRowLock", "checkAndDelete", + "checkAndDeleteAfterRowLock", "incrementColumnValue", "append", "appendAfterRowLock","increment", + "incrementAfterRowLock", "scan", "bulkLoadHFile", "prepareBulkLoad", + "cleanupBulkLoad"}; + private final String [] endpoint= {"invoke"}; + private final String [] accessController = {"grant", "revoke", "getUserPermissions"}; + private final String [] regionServer = {"stopRegionServer", "mergeRegions", "rollWALWriterRequest", "replicateLogEntries"}; + + @Override + public void init(){ + List cmds = new ArrayList<>(); + cmds.addAll(Arrays.asList(master)); + cmds.addAll(Arrays.asList(region)); + cmds.addAll(Arrays.asList(regionServer)); + cmds.addAll(Arrays.asList(endpoint)); + cmds.addAll(Arrays.asList(accessController)); + this.setCommands(cmds); + } + +} + http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseSensitivityTypeResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseSensitivityTypeResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseSensitivityTypeResolver.java new file mode 100644 index 0000000..3692150 --- /dev/null +++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseSensitivityTypeResolver.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eagle.service.security.hbase.resolver; + +import com.typesafe.config.Config; +import org.apache.eagle.metadata.service.ApplicationEntityService; +import org.apache.eagle.security.resolver.AbstractSensitivityTypeResolver; +import org.apache.eagle.security.service.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public class HBaseSensitivityTypeResolver extends AbstractSensitivityTypeResolver { + private final static Logger LOG = LoggerFactory.getLogger(HBaseSensitivityTypeResolver.class); + private ISecurityMetadataDAO dao; + + public HBaseSensitivityTypeResolver(ApplicationEntityService entityService, Config eagleServerConfig){ + // todo I know this is ugly, but push abstraction to later + dao = MetadataDaoFactory.getMetadataDAO(eagleServerConfig); + } + + @Override + public void init() { + Map> maps = getAllSensitivities(); + this.setSensitivityMaps(maps); + } + + private Map> getAllSensitivities(){ + Map> all = new HashMap<>(); + Collection entities = dao.listHBaseSensitivities(); + for(HBaseSensitivityEntity entity : entities){ + if(!all.containsKey(entity.getSite())){ + all.put(entity.getSite(), new HashMap<>()); + } + all.get(entity.getSite()).put(entity.getHBaseResource(), entity.getSensitivityType()); + } + return all; + } +} + http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseActionResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseActionResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseActionResolver.java deleted file mode 100644 index 3baa1da..0000000 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseActionResolver.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.service.security.hbase.resolver; - - -import com.typesafe.config.Config; -import org.apache.eagle.metadata.service.ApplicationEntityService; -import org.apache.eagle.security.resolver.AbstractCommandResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; - - -public class HbaseActionResolver extends AbstractCommandResolver { - public HbaseActionResolver(ApplicationEntityService entityService, Config eagleServerConfig){ - - } - private final static Logger LOG = LoggerFactory.getLogger(HbaseActionResolver.class); - private final String [] cmdStrs = {"ADMIN", "READ", "WRITE", "CREATE", "EXECUTE"}; - - @Override - public void init() { - this.setCommands(Arrays.asList(cmdStrs)); - } -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java deleted file mode 100644 index 6dd82e4..0000000 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.service.security.hbase.resolver; - -import com.typesafe.config.Config; -import org.apache.eagle.metadata.model.ApplicationEntity; -import org.apache.eagle.metadata.service.ApplicationEntityService; -import org.apache.eagle.service.alert.resolver.AttributeResolvable; -import org.apache.eagle.service.alert.resolver.AttributeResolveException; -import org.apache.eagle.service.alert.resolver.BadAttributeResolveRequestException; -import org.apache.eagle.service.alert.resolver.GenericAttributeResolveRequest; -import org.apache.eagle.service.security.hbase.HbaseMetadataBrowseWebResource; -import org.apache.eagle.service.security.hbase.dao.HbaseMetadataDAOImpl; -import org.apache.hadoop.conf.Configuration; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - - -public class HbaseMetadataResolver implements AttributeResolvable { - private ApplicationEntityService entityService; - public HbaseMetadataResolver(ApplicationEntityService entityService, Config eagleServerConfig){ - this.entityService = entityService; - } - - private Map getAppConfig(String site, String appType){ - ApplicationEntity entity = entityService.getBySiteIdAndAppType(site, appType); - return entity.getConfiguration(); - } - - private Configuration convert(Map originalConfig) throws Exception { - Configuration config = new Configuration(); - for (Map.Entry entry : originalConfig.entrySet()) { - config.set(entry.getKey().toString(), entry.getValue().toString()); - } - return config; - } - - @Override - public List resolve(GenericAttributeResolveRequest request) throws AttributeResolveException { - String query = request.getQuery().trim(); - String site = request.getSite().trim(); - String[] subResources = query.split(":"); - - try { - Map config = getAppConfig(site, HbaseMetadataBrowseWebResource.HBASE_APPLICATION); - Configuration conf = convert(config); - HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf); - - switch (subResources.length) { - case 1: - if(query.endsWith(":")) - return filterAndCombineAttribue(query, dao.getTables(subResources[0]), null); - else - return filterAndCombineAttribue("", dao.getNamespaces(), subResources[0]); - case 2: - if(query.endsWith(":")) - return filterAndCombineAttribue(query, dao.getColumnFamilies(subResources[1]), null); - else - return filterAndCombineAttribue(subResources[0], dao.getTables(subResources[0]), subResources[1]); - case 3: - return filterAndCombineAttribue(String.format("%s:%s", subResources[0], subResources[1]), dao.getColumnFamilies(subResources[1]), subResources[2]); - } - - } catch (IOException e) { - e.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - } - - return null; - } - - @Override - public void validateRequest(GenericAttributeResolveRequest request) throws BadAttributeResolveRequestException { - String query = request.getQuery(); - String site = request.getSite(); - if (query == null || query.startsWith(":") || !Pattern.matches("[a-zA-Z:]+", query) || site == null || site.length() == 0) { - throw new BadAttributeResolveRequestException("hBase resource must be {\"site\":\"${site}\", \"query\"=\"{namespace}:{table}:{columnFamily}\""); - } - } - - public List filterAndCombineAttribue(String prefix, List attrs, String target) { - List result = new ArrayList<>(); - String format = (prefix == "" ? "%s%s" : "%s:%s"); - - if(target == null) { - for (String attr : attrs){ - result.add(String.format("%s%s", prefix, attr)); - } - } else { - Pattern pattern = Pattern.compile("^" + target, Pattern.CASE_INSENSITIVE); - for (String attr : attrs) { - if (pattern.matcher(attr).find()) { - result.add(String.format(format, prefix, attr)); - } - } - if (result.size() == 0) { - for (String attr : attrs) { - result.add(String.format(format, prefix, attr)); - } - } - } - return result; - } - - @Override - public Class getRequestClass() { - return GenericAttributeResolveRequest.class; - } -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseRequestResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseRequestResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseRequestResolver.java deleted file mode 100644 index 8467f40..0000000 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseRequestResolver.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.service.security.hbase.resolver; - - -import com.typesafe.config.Config; -import org.apache.eagle.metadata.service.ApplicationEntityService; -import org.apache.eagle.security.resolver.AbstractCommandResolver; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - - -public class HbaseRequestResolver extends AbstractCommandResolver { - private final static Logger LOG = LoggerFactory.getLogger(HbaseRequestResolver.class); - - public HbaseRequestResolver(ApplicationEntityService entityService, Config eagleServerConfig){ - - } - - private final String [] master = {"createTable", "modifyTable", "deleteTable", "truncateTable", "addColumn", "modifyColumn", - "deleteColumn", "enableTable", "disableTable", "disableAclTable", "move", "assign", "unassign", - "regionOffline", "balance", "balanceSwitch", "shutdown", "stopMaster", "snapshot", - "listSnapshot", "cloneSnapshot", "restoreSnapshot", "deleteSnapshot", "createNamespace", - "deleteNamespace", "modifyNamespace", "getNamespaceDescriptor", "listNamespaceDescriptors*", - "flushTable", "getTableDescriptors*", "getTableNames*", "setUserQuota", "setTableQuota", - "setNamespaceQuota"}; - private final String [] region = {"openRegion", "closeRegion", "flush", "split", "compact", "getClosestRowBefore", "getOp", "exists", - "put", "delete", "batchMutate", "checkAndPut", "checkAndPutAfterRowLock", "checkAndDelete", - "checkAndDeleteAfterRowLock", "incrementColumnValue", "append", "appendAfterRowLock","increment", - "incrementAfterRowLock", "scan", "bulkLoadHFile", "prepareBulkLoad", - "cleanupBulkLoad"}; - private final String [] endpoint= {"invoke"}; - private final String [] accessController = {"grant", "revoke", "getUserPermissions"}; - private final String [] regionServer = {"stopRegionServer", "mergeRegions", "rollWALWriterRequest", "replicateLogEntries"}; - - @Override - public void init(){ - List cmds = new ArrayList<>(); - cmds.addAll(Arrays.asList(master)); - cmds.addAll(Arrays.asList(region)); - cmds.addAll(Arrays.asList(regionServer)); - cmds.addAll(Arrays.asList(endpoint)); - cmds.addAll(Arrays.asList(accessController)); - this.setCommands(cmds); - } - -} - http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseSensitivityTypeResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseSensitivityTypeResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseSensitivityTypeResolver.java deleted file mode 100644 index b5ad203..0000000 --- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseSensitivityTypeResolver.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eagle.service.security.hbase.resolver; - -import com.typesafe.config.Config; -import org.apache.eagle.metadata.service.ApplicationEntityService; -import org.apache.eagle.security.resolver.AbstractSensitivityTypeResolver; -import org.apache.eagle.security.service.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -public class HbaseSensitivityTypeResolver extends AbstractSensitivityTypeResolver { - private final static Logger LOG = LoggerFactory.getLogger(HbaseSensitivityTypeResolver.class); - private ISecurityMetadataDAO dao; - - public HbaseSensitivityTypeResolver(ApplicationEntityService entityService, Config eagleServerConfig){ - // todo I know this is ugly, but push abstraction to later - dao = MetadataDaoFactory.getMetadataDAO(eagleServerConfig); - } - - @Override - public void init() { - Map> maps = getAllSensitivities(); - this.setSensitivityMaps(maps); - } - - private Map> getAllSensitivities(){ - Map> all = new HashMap<>(); - Collection entities = dao.listHBaseSensitivities(); - for(HBaseSensitivityEntity entity : entities){ - if(!all.containsKey(entity.getSite())){ - all.put(entity.getSite(), new HashMap<>()); - } - all.get(entity.getSite()).put(entity.getHbaseResource(), entity.getSensitivityType()); - } - return all; - } -} - http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHBaseMetadataResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHBaseMetadataResolver.java b/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHBaseMetadataResolver.java new file mode 100644 index 0000000..9f90770 --- /dev/null +++ b/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHBaseMetadataResolver.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.service.security.hbase; + + +import org.junit.Test; + +public class TestHBaseMetadataResolver { + + @Test + public void testSplit() { + String query = "hbase:"; + String[] subResources = query.split(":"); + + query = "hbase:meta"; + subResources = query.split(":"); + + query = "hbase:meta:"; + subResources = query.split(":"); + + query = "hbase:meta:cf"; + subResources = query.split(":"); + + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHbaseMetadataResolver.java ---------------------------------------------------------------------- diff --git a/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHbaseMetadataResolver.java b/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHbaseMetadataResolver.java deleted file mode 100644 index 8cd0c32..0000000 --- a/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHbaseMetadataResolver.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.service.security.hbase; - - -import org.junit.Test; - -public class TestHbaseMetadataResolver { - - @Test - public void testSplit() { - String query = "hbase:"; - String[] subResources = query.split(":"); - - query = "hbase:meta"; - subResources = query.split(":"); - - query = "hbase:meta:"; - subResources = query.split(":"); - - query = "hbase:meta:cf"; - subResources = query.split(":"); - - } -} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java index 17dec20..22afeb3 100644 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java @@ -21,12 +21,11 @@ package org.apache.eagle.topology.extractor; import backtype.storm.spout.SpoutOutputCollector; import org.apache.eagle.topology.TopologyCheckAppConfig; import org.apache.eagle.topology.TopologyConstants; -import org.apache.eagle.topology.extractor.hbase.HbaseTopologyCrawler; +import org.apache.eagle.topology.extractor.hbase.HBaseTopologyCrawler; import org.apache.eagle.topology.extractor.hdfs.HdfsTopologyCrawler; import org.apache.eagle.topology.extractor.mr.MRTopologyCrawler; import org.apache.eagle.topology.resolver.TopologyRackResolver; -import org.apache.eagle.topology.resolver.impl.DefaultTopologyRackResolver; import org.slf4j.Logger; import java.lang.reflect.Constructor; @@ -69,7 +68,7 @@ public class TopologyExtractorFactory { } static { - registerTopologyExtractor(TopologyConstants.TopologyType.HBASE.name(), HbaseTopologyCrawler.class); + registerTopologyExtractor(TopologyConstants.TopologyType.HBASE.name(), HBaseTopologyCrawler.class); registerTopologyExtractor(TopologyConstants.TopologyType.HDFS.name(), HdfsTopologyCrawler.class); registerTopologyExtractor(TopologyConstants.TopologyType.MR.name(), MRTopologyCrawler.class); } http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyCrawler.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyCrawler.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyCrawler.java new file mode 100644 index 0000000..1ee1fa1 --- /dev/null +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyCrawler.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.topology.extractor.hbase; + +import backtype.storm.spout.SpoutOutputCollector; +import backtype.storm.tuple.Values; +import org.apache.eagle.topology.TopologyCheckAppConfig; +import org.apache.eagle.topology.TopologyCheckMessageId; +import org.apache.eagle.topology.TopologyConstants; +import org.apache.eagle.topology.extractor.TopologyEntityParserResult; +import org.apache.eagle.topology.extractor.TopologyCrawler; +import org.apache.eagle.topology.resolver.TopologyRackResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HBaseTopologyCrawler implements TopologyCrawler { + + private static final Logger LOG = LoggerFactory.getLogger(HBaseTopologyCrawler.class); + + private HBaseTopologyEntityParser parser; + private SpoutOutputCollector collector; + private String site; + + public HBaseTopologyCrawler(TopologyCheckAppConfig config, TopologyRackResolver rackResolver, SpoutOutputCollector collector) { + this.site = config.dataExtractorConfig.site; + this.parser = new HBaseTopologyEntityParser(this.site, config.hBaseConfig, rackResolver); + this.collector = collector; + } + + @Override + public void extract() { + long updateTimestamp = System.currentTimeMillis(); + TopologyEntityParserResult result = parser.parse(updateTimestamp);; + + if (result == null || result.getMetrics().isEmpty()) { + LOG.warn("No data fetched"); + result = new TopologyEntityParserResult(); + } + TopologyCheckMessageId messageId = new TopologyCheckMessageId(TopologyConstants.TopologyType.HBASE, updateTimestamp); + this.collector.emit(new Values(TopologyConstants.HBASE_INSTANCE_SERVICE_NAME, result), messageId); + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyEntityParser.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyEntityParser.java new file mode 100644 index 0000000..a73fbf8 --- /dev/null +++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyEntityParser.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.eagle.topology.extractor.hbase; + +import org.apache.eagle.app.utils.HadoopSecurityUtil; +import org.apache.eagle.topology.TopologyCheckAppConfig; +import org.apache.eagle.topology.TopologyConstants; +import org.apache.eagle.topology.extractor.TopologyEntityParserResult; +import org.apache.eagle.topology.entity.HBaseServiceTopologyAPIEntity; +import org.apache.eagle.topology.extractor.TopologyEntityParser; +import org.apache.eagle.topology.resolver.TopologyRackResolver; +import org.apache.eagle.topology.utils.EntityBuilderHelper; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.ServerLoad; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.slf4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.eagle.topology.TopologyConstants.*; + +public class HBaseTopologyEntityParser implements TopologyEntityParser { + + private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(HBaseTopologyEntityParser.class); + private Configuration hBaseConfiguration; + private String site; + private Boolean kerberosEnable = false; + private TopologyRackResolver rackResolver; + + public HBaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig, TopologyRackResolver resolver) { + this.site = site; + this.rackResolver = resolver; + this.hBaseConfiguration = HBaseConfiguration.create(); + this.hBaseConfiguration.set("hbase.zookeeper.quorum", hBaseConfig.zkQuorum); + this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", hBaseConfig.zkClientPort); + this.hBaseConfiguration.set("zookeeper.znode.parent", hBaseConfig.zkRoot); + this.hBaseConfiguration.set("hbase.client.retries.number", hBaseConfig.zkRetryTimes); + // kerberos authentication + if (hBaseConfig.eaglePrincipal != null && hBaseConfig.eagleKeytab != null + && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) { + this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal); + this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab); + this.kerberosEnable = true; + this.hBaseConfiguration.set("hbase.security.authentication", "kerberos"); + this.hBaseConfiguration.set("hbase.master.kerberos.principal", hBaseConfig.hbaseMasterPrincipal); + } + } + + private HBaseAdmin getHBaseAdmin() throws IOException { + if (this.kerberosEnable) { + HadoopSecurityUtil.login(hBaseConfiguration); + } + return new HBaseAdmin(this.hBaseConfiguration); + } + + + @Override + public TopologyEntityParserResult parse(long timestamp) { + final TopologyEntityParserResult result = new TopologyEntityParserResult(); + int activeRatio = 0; + try { + doParse(timestamp, result); + activeRatio++; + } catch (Exception ex) { + LOG.error(ex.getMessage(), ex); + } + result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.HMASTER_ROLE, activeRatio, site, timestamp)); + return result; + } + + private void doParse(long timestamp, TopologyEntityParserResult result) throws IOException { + long deadServers = 0; + long liveServers = 0; + HBaseAdmin admin = null; + try { + admin = getHBaseAdmin(); + ClusterStatus status = admin.getClusterStatus(); + deadServers = status.getDeadServers(); + liveServers = status.getServersSize(); + + for (ServerName liveServer : status.getServers()) { + ServerLoad load = status.getLoad(liveServer); + result.getSlaveNodes().add(parseServer(liveServer, load, TopologyConstants.REGIONSERVER_ROLE, TopologyConstants.REGIONSERVER_LIVE_STATUS, timestamp)); + } + for (ServerName deadServer : status.getDeadServerNames()) { + ServerLoad load = status.getLoad(deadServer); + result.getSlaveNodes().add(parseServer(deadServer, load, TopologyConstants.REGIONSERVER_ROLE, TopologyConstants.REGIONSERVER_DEAD_STATUS, timestamp)); + } + ServerName master = status.getMaster(); + if (master != null) { + ServerLoad load = status.getLoad(master); + result.getMasterNodes().add(parseServer(master, load, TopologyConstants.HMASTER_ROLE, TopologyConstants.HMASTER_ACTIVE_STATUS, timestamp)); + } + for (ServerName backupMaster : status.getBackupMasters()) { + ServerLoad load = status.getLoad(backupMaster); + result.getMasterNodes().add(parseServer(backupMaster, load, TopologyConstants.HMASTER_ROLE, TopologyConstants.HMASTER_STANDBY_STATUS, timestamp)); + } + double liveRatio = liveServers * 1d / (liveServers + deadServers); + result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.REGIONSERVER_ROLE, liveRatio, site, timestamp)); + LOG.info("live servers: {}, dead servers: {}", liveServers, deadServers); + } finally { + if (admin != null) { + try { + admin.close(); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + } + } + + private HBaseServiceTopologyAPIEntity parseServer(ServerName serverName, ServerLoad serverLoad, String role, String status, long timestamp) { + if (serverName == null) { + return null; + } + HBaseServiceTopologyAPIEntity entity = createEntity(role, serverName.getHostname(), timestamp); + parseServerLoad(entity, serverLoad); + entity.setStatus(status); + return entity; + } + + private void parseServerLoad(HBaseServiceTopologyAPIEntity entity, ServerLoad load) { + if (load == null) { + return; + } + entity.setMaxHeapMB(load.getMaxHeapMB()); + entity.setUsedHeapMB(load.getUsedHeapMB()); + entity.setNumRegions(load.getNumberOfRegions()); + entity.setNumRequests(load.getNumberOfRequests()); + } + + private HBaseServiceTopologyAPIEntity createEntity(String roleType, String hostname, long timestamp) { + HBaseServiceTopologyAPIEntity entity = new HBaseServiceTopologyAPIEntity(); + Map tags = new HashMap(); + entity.setTags(tags); + tags.put(SITE_TAG, site); + tags.put(ROLE_TAG, roleType); + tags.put(HOSTNAME_TAG, hostname); + String rack = rackResolver.resolve(hostname); + tags.put(RACK_TAG, rack); + entity.setLastUpdateTime(timestamp); + entity.setTimestamp(timestamp); + return entity; + } + + @Override + public TopologyConstants.TopologyType getTopologyType() { + return TopologyType.HBASE; + } + + @Override + public TopologyConstants.HadoopVersion getHadoopVersion() { + return HadoopVersion.V2; + } +} http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java ---------------------------------------------------------------------- diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java deleted file mode 100644 index 04a4aa1..0000000 --- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.eagle.topology.extractor.hbase; - -import backtype.storm.spout.SpoutOutputCollector; -import backtype.storm.tuple.Values; -import org.apache.eagle.topology.TopologyCheckAppConfig; -import org.apache.eagle.topology.TopologyCheckMessageId; -import org.apache.eagle.topology.TopologyConstants; -import org.apache.eagle.topology.extractor.TopologyEntityParserResult; -import org.apache.eagle.topology.extractor.TopologyCrawler; -import org.apache.eagle.topology.resolver.TopologyRackResolver; -import org.apache.eagle.topology.utils.EntityBuilderHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -public class HbaseTopologyCrawler implements TopologyCrawler { - - private static final Logger LOG = LoggerFactory.getLogger(HbaseTopologyCrawler.class); - - private HbaseTopologyEntityParser parser; - private SpoutOutputCollector collector; - private String site; - - public HbaseTopologyCrawler(TopologyCheckAppConfig config, TopologyRackResolver rackResolver, SpoutOutputCollector collector) { - this.site = config.dataExtractorConfig.site; - this.parser = new HbaseTopologyEntityParser(this.site, config.hBaseConfig, rackResolver); - this.collector = collector; - } - - @Override - public void extract() { - long updateTimestamp = System.currentTimeMillis(); - TopologyEntityParserResult result = parser.parse(updateTimestamp);; - - if (result == null || result.getMetrics().isEmpty()) { - LOG.warn("No data fetched"); - result = new TopologyEntityParserResult(); - } - TopologyCheckMessageId messageId = new TopologyCheckMessageId(TopologyConstants.TopologyType.HBASE, updateTimestamp); - this.collector.emit(new Values(TopologyConstants.HBASE_INSTANCE_SERVICE_NAME, result), messageId); - } -}