ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From v..@apache.org
Subject [1/2] ranger git commit: RANGER-1271 Ranger-Atlas testConnection feature
Date Fri, 03 Feb 2017 04:30:32 GMT
Repository: ranger
Updated Branches:
  refs/heads/master 4603dfa3d -> 9fea06dbb


RANGER-1271 Ranger-Atlas testConnection feature

Signed-off-by: Velmurugan Periasamy <vel@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/306075df
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/306075df
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/306075df

Branch: refs/heads/master
Commit: 306075df553e295041ef5b6e5b01a8a6799c9654
Parents: 4603dfa
Author: zhangqiang2 <zhangqiang2@zte.com.cn>
Authored: Thu Jan 5 23:15:12 2017 +0800
Committer: Velmurugan Periasamy <vel@apache.org>
Committed: Thu Feb 2 23:29:53 2017 -0500

----------------------------------------------------------------------
 .../services/atlas/RangerServiceAtlas.java      |  50 +++-
 .../services/atlas/client/AtlasClient.java      | 292 +++++++++++++++++++
 .../atlas/client/AtlasConnectionMgr.java        |  30 ++
 .../services/atlas/client/AtlasResourceMgr.java |  88 ++++++
 src/main/assembly/admin-web.xml                 |  16 +-
 5 files changed, 464 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java
----------------------------------------------------------------------
diff --git a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java
b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java
index cb5a7eb..ba4d263 100644
--- a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java
+++ b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/RangerServiceAtlas.java
@@ -21,14 +21,15 @@ package org.apache.ranger.services.atlas;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
-
+import java.util.Map;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.ranger.plugin.client.BaseClient;
 import org.apache.ranger.plugin.model.RangerService;
 import org.apache.ranger.plugin.model.RangerServiceDef;
 import org.apache.ranger.plugin.service.RangerBaseService;
 import org.apache.ranger.plugin.service.ResourceLookupContext;
+import org.apache.ranger.services.atlas.client.AtlasResourceMgr;
+
 
 public class RangerServiceAtlas extends RangerBaseService {
 
@@ -44,19 +45,46 @@ public class RangerServiceAtlas extends RangerBaseService {
     }
 
     @Override
-    public HashMap<String, Object> validateConfig() throws Exception {
-        if (LOG.isDebugEnabled()) {
-            LOG.debug("This method will be implemented");
-        }
+    public HashMap<String,Object> validateConfig() throws Exception {
         HashMap<String, Object> responseMap = new HashMap<String, Object>();
-        String msg = "This feature is not available currently";
-        BaseClient.generateResponseDataMap(true, msg, msg, null, null, responseMap);
+            String serviceName = getServiceName();
+                if(LOG.isDebugEnabled()) {
+                       LOG.debug("==> RangerServiceAtlas.validateConfig Service: (" +
serviceName + " )");
+                  }
+                if ( configs != null) {
+                       try  {
+                                responseMap = AtlasResourceMgr.validateConfig(serviceName,
configs);
+                        } catch (Exception e) {
+                                LOG.error("<== RangerServiceAtlas.validateConfig Error:"
+ e);
+                                throw e;
+                        }
+                }
+                if(LOG.isDebugEnabled()) {
+                        LOG.debug("<== RangerServiceAtlas.validateConfig Response : ("
+ responseMap + " )");
+                }
         return responseMap;
     }
 
+
     @Override
     public List<String> lookupResource(ResourceLookupContext context) throws Exception
{
-        //This feature is not available currently
-        return new ArrayList<String>();
-    }
+                List<String> ret = new ArrayList<String>();
+                String serviceName = getServiceName();
+                Map<String,String> configs = getConfigs();
+                if(LOG.isDebugEnabled()) {
+                        LOG.debug("==> RangerServiceAtlas.lookupResource Context: (" +
context + ")");
+                }
+                if (context != null) {
+                        try {
+                               ret  = AtlasResourceMgr.getAtlasTermResources(serviceName,configs,context);
+                        } catch (Exception e) {
+                                LOG.error( "<==RangerServiceAtlas.lookupResource Error
: " + e);
+                        throw e;
+                        }
+                }
+                if(LOG.isDebugEnabled()) {
+                        LOG.debug("<== RangerServiceAtlas.lookupResource Response: ("
+ ret + ")");
+                }
+                return ret;
+     }
 }

http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasClient.java
----------------------------------------------------------------------
diff --git a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasClient.java
b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasClient.java
new file mode 100644
index 0000000..4f90469
--- /dev/null
+++ b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasClient.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.services.atlas.client;
+
+import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import javax.security.auth.Subject;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.NewCookie;
+
+import org.apache.log4j.Logger;
+import org.apache.ranger.plugin.client.BaseClient;
+import org.apache.ranger.plugin.client.HadoopException;
+import org.apache.ranger.plugin.util.PasswordUtils;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.core.util.MultivaluedMapImpl;
+
+public class AtlasClient extends BaseClient {
+
+	private static final Logger LOG = Logger.getLogger(AtlasClient.class);
+	private static final String EXPECTED_MIME_TYPE = "application/json";
+	private static final String ATLAS_STATUS_API_ENDPOINT = "/j_spring_security_check";
+	private static final String ATLAS_LIST_TERM_API_ENDPOINT = "/api/atlas/types";
+	private static final String errMessage =  " You can still save the repository and start
creating "
+											  + "policies, but you would not be able to use autocomplete for "
+											  + "resource names. Check ranger_admin.log for more info.";
+
+	private String atlasUrl;
+	private String userName;
+	private String password;
+
+	public  AtlasClient(String serviceName, Map<String, String> configs) {
+
+		super(serviceName,configs,"atlas-client");
+
+		this.atlasUrl = configs.get("atlas.rest.address");
+		this.userName = configs.get("username");
+		this.password = configs.get("password");
+		if (this.atlasUrl == null || this.atlasUrl.isEmpty()) {
+			LOG.error("No value found for configuration 'atlas.rest.address'. Atlas resource lookup
will fail");
+        }
+		if (this.userName == null || this.userName.isEmpty()) {
+            LOG.error("No value found for configuration 'usename'. Atlas resource lookup
will fail");
+        }
+		if (this.password == null || this.password.isEmpty()) {
+            LOG.error("No value found for configuration 'password'. Atlas resource lookup
will fail");
+        }
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Atlas Client is build with url [" + this.atlasUrl + "] user: [" + this.userName
+ "], password: [" + "*********" + "]");
+		}
+	}
+
+	public List<String> getTermList( String termNameMatching, List<String> existingTermList)
{
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Getting Atlas Terms list for termNameMatching : " + termNameMatching);
+		}
+		final String errMsg = errMessage;
+		List<String> ret = null;
+
+		Callable<List<String>> callableAtlasTermListGetter = new Callable<List<String>>()
{
+
+			@Override
+			public List<String> call() {
+				List<String> atlasTermListGetter = null;
+
+				Subject subj = getLoginSubject();
+
+				if (subj != null) {
+					atlasTermListGetter = Subject.doAs(subj, new PrivilegedAction<List<String>>()
{
+
+					@Override
+					public List<String> run() {
+
+						List<String> lret = new ArrayList<String>();
+
+						String statusUrl = atlasUrl + ATLAS_STATUS_API_ENDPOINT;
+						String resultUrl = atlasUrl + ATLAS_LIST_TERM_API_ENDPOINT;
+
+						Client client = null;
+						ClientResponse statusResponse = null;
+						ClientResponse resultResponse = null;
+
+						try {
+							client = Client.create();
+							WebResource webResource = client.resource(statusUrl);
+							MultivaluedMap<String, String> formData = new MultivaluedMapImpl();
+							formData.add("j_username", userName);
+							formData.add("j_password",PasswordUtils.decryptPassword(password));
+							try {
+								statusResponse = webResource.type("application/x-www-form-urlencoded").post(
+										ClientResponse.class, formData);
+							} catch (Exception e) {
+								String msgDesc = "Unable to get a valid statusResponse for "
+										+ "expected mime type : [" + EXPECTED_MIME_TYPE
+										+ "] URL : " + statusUrl + " - got null response.";
+								LOG.error(msgDesc);
+							}
+
+							if (LOG.isDebugEnabled()) {
+								LOG.debug("getTermList():calling " + statusUrl);
+							}
+
+							if (statusResponse != null) {
+								if (LOG.isDebugEnabled()) {
+								LOG.debug("getTermList():response.getStatus()= " + statusResponse.getStatus());
+                                                               }
+								if (statusResponse.getStatus() == 200) {
+									WebResource webResource2 = client
+											.resource(resultUrl);
+									WebResource.Builder builder = webResource2.getRequestBuilder();
+									for (NewCookie cook : statusResponse.getCookies()) {			                        
                                              builder = builder.cookie(cook);
+									}
+									resultResponse = builder.get(ClientResponse.class);
+									lret.add(resultResponse.getEntity(String.class));
+								} else{
+									LOG.info("getTermList():response.getStatus()= " + statusResponse.getStatus() + "
for URL " + statusUrl + ", so returning null list");
+									LOG.info(statusResponse.getEntity(String.class));
+									lret = null;
+								}
+							}
+						}  catch (Throwable t) {
+							lret = null;
+							String msgDesc = "Exception while getting Atlas Term List."
+									+ " URL : " + statusUrl;
+							HadoopException hdpException = new HadoopException(msgDesc,
+										t);
+							LOG.error(msgDesc, t);
+							hdpException.generateResponseDataMap(false,
+									BaseClient.getMessage(t), msgDesc + errMsg, null,
+									null);
+							throw hdpException;
+
+						} finally {
+							if (statusResponse != null) {
+								statusResponse.close();
+							}
+							if (resultResponse != null) {
+								resultResponse.close();
+							}
+
+							if (client != null) {
+								client.destroy();
+							}
+						}
+						return lret;
+					}
+				  } );
+				}
+				return atlasTermListGetter;
+			  }
+			};
+		try {
+			ret = timedTask(callableAtlasTermListGetter, 5, TimeUnit.SECONDS);
+		} catch ( Throwable t) {
+			LOG.error("Unable to get Atlas Terms list from [" + atlasUrl + "]", t);
+			String msgDesc = "Unable to get a valid response for "
+					+ "expected mime type : [" + EXPECTED_MIME_TYPE
+					+ "] URL : " + atlasUrl;
+			HadoopException hdpException = new HadoopException(msgDesc,
+					t);
+			LOG.error(msgDesc, t);
+
+			hdpException.generateResponseDataMap(false,
+					BaseClient.getMessage(t), msgDesc + errMsg, null,
+					null);
+			throw hdpException;
+		}
+		return ret;
+	}
+
+	public static HashMap<String, Object> connectionTest(String serviceName,
+			Map<String, String> configs) {
+
+		String errMsg = errMessage;
+		boolean connectivityStatus = false;
+		HashMap<String, Object> responseData = new HashMap<String, Object>();
+
+		AtlasClient AtlasClient = getAtlasClient(serviceName,
+				configs);
+		List<String> strList = getAtlasTermResource(AtlasClient, "",null);
+
+		if (strList != null && strList.size() > 0 ) {
+			if (LOG.isDebugEnabled()) {
+				LOG.debug("TESTING Term list size" + strList.size() + " Atlas Terms");
+			}
+			connectivityStatus = true;
+		}
+
+		if (connectivityStatus) {
+			String successMsg = "ConnectionTest Successful";
+			BaseClient.generateResponseDataMap(connectivityStatus, successMsg,
+					successMsg, null, null, responseData);
+		} else {
+			String failureMsg = "Unable to retrieve any Atlas Terms using given parameters.";
+			BaseClient.generateResponseDataMap(connectivityStatus, failureMsg,
+					failureMsg + errMsg, null, null, responseData);
+		}
+
+		return responseData;
+	}
+
+	public static AtlasClient getAtlasClient(String serviceName,
+			Map<String, String> configs) {
+		AtlasClient AtlasClient = null;
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("Getting AtlasClient for datasource: " + serviceName);
+		}
+		String errMsg = errMessage;
+		if (configs == null || configs.isEmpty()) {
+			String msgDesc = "Could not connect as Connection ConfigMap is empty.";
+			LOG.error(msgDesc);
+			HadoopException hdpException = new HadoopException(msgDesc);
+			hdpException.generateResponseDataMap(false, msgDesc, msgDesc
+					+ errMsg, null, null);
+			throw hdpException;
+		} else {
+			AtlasClient = new AtlasClient (serviceName, configs);
+		}
+		return AtlasClient;
+	}
+
+	public static List<String> getAtlasTermResource (final AtlasClient atlasClient,
+			String atlasTermName, List<String> existingAtlasTermName) {
+
+		List<String> resultList = new ArrayList<String>();
+		String errMsg = errMessage;
+
+		try {
+			if (atlasClient == null) {
+				String msgDesc = "Unable to get Atlas Terms : AtlasClient is null.";
+				LOG.error(msgDesc);
+				HadoopException hdpException = new HadoopException(msgDesc);
+				hdpException.generateResponseDataMap(false, msgDesc, msgDesc
+						+ errMsg, null, null);
+				throw hdpException;
+			}
+
+			if (atlasTermName != null) {
+				String finalAtlasTermName = atlasTermName.trim();
+				resultList = atlasClient
+						.getTermList(finalAtlasTermName,existingAtlasTermName);
+				if (resultList != null) {
+					if (LOG.isDebugEnabled()) {
+						LOG.debug("Returning list of " + resultList.size() + " Atlas Terms");
+					}
+				}
+			}
+		}catch (Throwable t) {
+			String msgDesc = "getAtlasResource: Unable to get Atlas resources.";
+			LOG.error(msgDesc, t);
+			HadoopException hdpException = new HadoopException(msgDesc);
+
+			hdpException.generateResponseDataMap(false,
+					BaseClient.getMessage(t), msgDesc + errMsg, null, null);
+			throw hdpException;
+		}
+
+		return resultList;
+	}
+
+	public static <T> T timedTask(Callable<T> callableObj, long timeout,
+			TimeUnit timeUnit) throws Exception {
+		return callableObj.call();
+	}
+}

http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasConnectionMgr.java
----------------------------------------------------------------------
diff --git a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasConnectionMgr.java
b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasConnectionMgr.java
new file mode 100644
index 0000000..84e24c5
--- /dev/null
+++ b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasConnectionMgr.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.services.atlas.client;
+
+import java.util.Map;
+
+public class AtlasConnectionMgr {
+
+	public static AtlasClient getAtlasClient(String serviceName, Map<String, String> configs)
{
+        return  new AtlasClient(serviceName, configs);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasResourceMgr.java
----------------------------------------------------------------------
diff --git a/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasResourceMgr.java
b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasResourceMgr.java
new file mode 100644
index 0000000..f738601
--- /dev/null
+++ b/plugin-atlas/src/main/java/org/apache/ranger/services/atlas/client/AtlasResourceMgr.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.services.atlas.client;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.apache.ranger.plugin.service.ResourceLookupContext;
+
+
+public class AtlasResourceMgr {
+	public static final Logger LOG = Logger.getLogger(AtlasResourceMgr.class);
+	private static final String  ATLASTERM	= "terms";
+
+	public static HashMap<String, Object> validateConfig(String serviceName, Map<String,
String> configs) throws Exception {
+
+		HashMap<String, Object> ret = null;
+
+		if(LOG.isDebugEnabled()) {
+			LOG.debug("==> AtlasResourceMgr.validateConfig ServiceName: "+ serviceName + "Configs"
+ configs );
+		}
+
+		try {
+			ret = AtlasClient.connectionTest(serviceName, configs);
+		} catch (Exception e) {
+			LOG.error("<== AtlasResourceMgr.validateConfig Error: " + e);
+		  throw e;
+		}
+
+		if(LOG.isDebugEnabled()) {
+			LOG.debug("<== AtlasResourceMgr.validateConfig Result : "+ ret  );
+		}
+		return ret;
+	}
+
+    public static List<String> getAtlasTermResources(String serviceName, Map<String,
String> configs,ResourceLookupContext context) {
+           String userInput = context.getUserInput();
+	   Map<String, List<String>> resourceMap = context.getResources();
+	   List<String> resultList = null;
+	   List<String> atlasTermList = null;
+           String  atlasTermName = null;
+
+		if ( resourceMap != null && !resourceMap.isEmpty() &&
+			resourceMap.get(ATLASTERM) != null ) {
+			atlasTermName = userInput;
+			atlasTermList = resourceMap.get(ATLASTERM);
+		} else {
+			atlasTermName = userInput;
+		}
+
+        if (configs == null || configs.isEmpty()) {
+                LOG.error("Connection Config is empty");
+        } else {
+               resultList = getAtlasTermResource(serviceName, configs, atlasTermName,atlasTermList);
+        }
+        return resultList;
+    }
+
+    public static List<String> getAtlasTermResource(String serviceName, Map<String,
String> configs, String atlasTermName, List<String> atlasTermList) {
+        final AtlasClient atlasClient = AtlasConnectionMgr.getAtlasClient(serviceName, configs);
+        List<String> termList = null;
+	    if (atlasClient != null) {
+	       synchronized(atlasClient) {
+               termList = atlasClient.getTermList(atlasTermName, atlasTermList);
+             }
+          }
+       return termList;
+     }
+}

http://git-wip-us.apache.org/repos/asf/ranger/blob/306075df/src/main/assembly/admin-web.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml
index 4779071..966033f 100644
--- a/src/main/assembly/admin-web.xml
+++ b/src/main/assembly/admin-web.xml
@@ -310,7 +310,21 @@
               <include>org.apache.ranger:ranger-nifi-plugin</include>
           </includes>
       </moduleSet>
-  </moduleSets> 	
+
+      <moduleSet>
+          <binaries>
+              <includeDependencies>true</includeDependencies>
+              <outputDirectory>/ews/webapp/WEB-INF/classes/ranger-plugins/atlas</outputDirectory>
+              <unpack>false</unpack>
+              <directoryMode>755</directoryMode>
+              <fileMode>644</fileMode>
+          </binaries>
+          <includes>
+              <include>org.apache.ranger:ranger-atlas-plugin</include>
+          </includes>
+      </moduleSet>
+
+  </moduleSets>
 
   <fileSets>
 	<fileSet>


Mime
View raw message