ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cohei...@apache.org
Subject ranger git commit: RANGER-1940 - Upgrade to Knox 1.0.0
Date Wed, 07 Feb 2018 15:02:37 GMT
Repository: ranger
Updated Branches:
  refs/heads/master f5c15d40b -> 2de803d12


RANGER-1940 - Upgrade to Knox 1.0.0


Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/2de803d1
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/2de803d1
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/2de803d1

Branch: refs/heads/master
Commit: 2de803d1241244a38d30cdb762df3b2608b045da
Parents: f5c15d4
Author: Colm O hEigeartaigh <coheigea@apache.org>
Authored: Wed Feb 7 15:02:14 2018 +0000
Committer: Colm O hEigeartaigh <coheigea@apache.org>
Committed: Wed Feb 7 15:02:14 2018 +0000

----------------------------------------------------------------------
 .../authorization/knox/RangerPDPKnoxFilter.java |   8 +-
 .../ranger/services/knox/KnoxRangerTest.java    | 142 +++++--------------
 pom.xml                                         |   2 +-
 .../RangerPDPKnoxDeploymentContributor.java     |  12 +-
 ...gateway.deploy.ProviderDeploymentContributor |  18 ---
 ...gateway.deploy.ProviderDeploymentContributor |  18 +++
 6 files changed, 61 insertions(+), 139 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java
----------------------------------------------------------------------
diff --git a/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java
b/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java
index 24e8702..f84a3e0 100644
--- a/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java
+++ b/knox-agent/src/main/java/org/apache/ranger/authorization/knox/RangerPDPKnoxFilter.java
@@ -35,10 +35,10 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.gateway.filter.AbstractGatewayFilter;
-import org.apache.hadoop.gateway.security.GroupPrincipal;
-import org.apache.hadoop.gateway.security.ImpersonatedPrincipal;
-import org.apache.hadoop.gateway.security.PrimaryPrincipal;
+import org.apache.knox.gateway.filter.AbstractGatewayFilter;
+import org.apache.knox.gateway.security.GroupPrincipal;
+import org.apache.knox.gateway.security.ImpersonatedPrincipal;
+import org.apache.knox.gateway.security.PrimaryPrincipal;
 import org.apache.ranger.audit.provider.MiscUtil;
 import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
 import org.apache.ranger.plugin.policyengine.RangerAccessResult;

http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java
----------------------------------------------------------------------
diff --git a/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java
b/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java
index 53e66df..7641921 100644
--- a/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java
+++ b/knox-agent/src/test/java/org/apache/ranger/services/knox/KnoxRangerTest.java
@@ -21,26 +21,15 @@ import static io.restassured.RestAssured.given;
 import static org.hamcrest.CoreMatchers.is;
 
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
-import org.apache.directory.server.protocol.shared.transport.TcpTransport;
-import org.apache.hadoop.gateway.GatewayServer;
-import org.apache.hadoop.gateway.GatewayTestConfig;
-import org.apache.hadoop.gateway.security.ldap.SimpleLdapDirectoryServer;
-import org.apache.hadoop.gateway.services.DefaultGatewayServices;
-import org.apache.hadoop.gateway.services.ServiceLifecycleException;
-import org.apache.hadoop.test.mock.MockServer;
+import org.apache.knox.gateway.GatewayTestConfig;
+import org.apache.knox.gateway.GatewayTestDriver;
 import org.apache.http.HttpStatus;
 import org.junit.AfterClass;
-import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -55,91 +44,25 @@ import io.restassured.response.ValidatableResponse;
  */
 public class KnoxRangerTest {
 
-    private static GatewayTestConfig config;
-    private static GatewayServer gateway;
-    private static SimpleLdapDirectoryServer ldap;
-    private static TcpTransport ldapTransport;
-    private static MockServer hdfsServer;
-    private static MockServer stormServer;
-    private static MockServer hbaseServer;
-    private static MockServer kafkaServer;
-    private static MockServer solrServer;
+    private static GatewayTestDriver driver = new GatewayTestDriver();
 
     @BeforeClass
     public static void setupSuite() throws Exception {
-        setupLdap();
-        hdfsServer = new MockServer( "hdfs", true );
-        stormServer = new MockServer( "storm", true );
-        hbaseServer = new MockServer( "hbase", true );
-        kafkaServer = new MockServer( "kafka", true );
-        solrServer = new MockServer( "solr", true );
-
-        setupGateway();
+        driver.setResourceBase(KnoxRangerTest.class);
+        driver.setupLdap(0);
+        GatewayTestConfig config = new GatewayTestConfig();
+        driver.setupService("WEBHDFS", "http://localhost:50070/webhdfs", "/cluster/webhdfs",
true);
+        driver.setupService("STORM", "http://localhost:8477", "/cluster/storm", true);
+        driver.setupService("SOLR", "http://localhost:8983", "/cluster/solr", true);
+        driver.setupService("WEBHBASE", "http://localhost:60080", "/cluster/hbase", true);
+        driver.setupService("KAFKA", "http://localhost:8477", "/cluster/kafka", true);
+
+        driver.setupGateway(config, "cluster", createTopology(), true);
     }
 
     @AfterClass
     public static void cleanupSuite() throws Exception {
-        gateway.stop();
-
-        FileUtils.deleteQuietly( new File( config.getGatewayTopologyDir() ) );
-        FileUtils.deleteQuietly( new File( config.getGatewayConfDir() ) );
-        FileUtils.deleteQuietly( new File( config.getGatewaySecurityDir() ) );
-        FileUtils.deleteQuietly( new File( config.getGatewayDeploymentDir() ) );
-        FileUtils.deleteQuietly( new File( config.getGatewayDataDir() ) );
-
-        hdfsServer.stop();
-        stormServer.stop();
-        hbaseServer.stop();
-        kafkaServer.stop();
-        solrServer.stop();
-
-        ldap.stop( true );
-    }
-
-    private static void setupLdap() throws Exception {
-        String basedir = System.getProperty("basedir");
-        if (basedir == null) {
-            basedir = new File(".").getCanonicalPath();
-        }
-        Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/users.ldif");
-        ldapTransport = new TcpTransport( 0 );
-        ldap = new SimpleLdapDirectoryServer( "dc=hadoop,dc=apache,dc=org", path.toFile(),
ldapTransport );
-        ldap.start();
-    }
-
-    private static void setupGateway() throws Exception {
-
-        File targetDir = new File( System.getProperty( "user.dir" ), "target" );
-        File gatewayDir = new File( targetDir, "gateway-home-" + UUID.randomUUID() );
-        Assert.assertTrue(gatewayDir.mkdirs());
-
-        config = new GatewayTestConfig();
-        config.setGatewayHomeDir( gatewayDir.getAbsolutePath() );
-
-        config.setGatewayServicesDir(targetDir.getPath() + File.separator + "services");
-
-        File topoDir = new File( config.getGatewayTopologyDir() );
-        Assert.assertTrue(topoDir.mkdirs());
-
-        File deployDir = new File( config.getGatewayDeploymentDir() );
-        Assert.assertTrue(deployDir.mkdirs());
-
-        File descriptor = new File( topoDir, "cluster.xml" );
-        FileOutputStream stream = new FileOutputStream( descriptor );
-        createTopology().toStream( stream );
-        stream.close();
-
-        DefaultGatewayServices srvcs = new DefaultGatewayServices();
-        Map<String,String> options = new HashMap<>();
-        options.put( "persist-master", "false" );
-        options.put( "master", "password" );
-        try {
-            srvcs.init( config, options );
-        } catch ( ServiceLifecycleException e ) {
-            e.printStackTrace(); // I18N not required.
-        }
-
-        gateway = GatewayServer.startGateway( config, srvcs );
+        driver.cleanup();
     }
 
     /**
@@ -170,8 +93,7 @@ public class KnoxRangerTest {
             .addTag( "value" ).addText( "uid={0},ou=people,dc=hadoop,dc=apache,dc=org" ).gotoParent()
             .addTag( "param" )
             .addTag( "name" ).addText( "main.ldapRealm.contextFactory.url" )
-            .addTag( "value" ).addText( "ldap://localhost:" + ldapTransport.getAcceptor().getLocalAddress().getPort()
).gotoParent()
-            //.addTag( "value" ).addText(driver.getLdapUrl() ).gotoParent()
+            .addTag( "value" ).addText(driver.getLdapUrl() ).gotoParent()
             .addTag( "param" )
             .addTag( "name" ).addText( "main.ldapRealm.contextFactory.authenticationMechanism"
)
             .addTag( "value" ).addText( "simple" ).gotoParent()
@@ -189,19 +111,19 @@ public class KnoxRangerTest {
             .gotoRoot()
             .addTag("service")
             .addTag("role").addText("WEBHDFS")
-            .addTag("url").addText("http://localhost:" + hdfsServer.getPort()).gotoParent()
+            .addTag("url").addText(driver.getRealUrl("WEBHDFS")).gotoParent()
             .addTag("service")
             .addTag("role").addText("STORM")
-            .addTag("url").addText("http://localhost:" + stormServer.getPort()).gotoParent()
+            .addTag("url").addText(driver.getRealUrl("STORM")).gotoParent()
             .addTag("service")
             .addTag("role").addText("WEBHBASE")
-            .addTag("url").addText("http://localhost:" + hbaseServer.getPort()).gotoParent()
+            .addTag("url").addText(driver.getRealUrl("WEBHBASE")).gotoParent()
             .addTag("service")
             .addTag("role").addText("KAFKA")
-            .addTag("url").addText("http://localhost:" + kafkaServer.getPort()).gotoParent()
+            .addTag("url").addText(driver.getRealUrl("KAFKA")).gotoParent()
             .addTag("service")
             .addTag("role").addText("SOLR")
-            .addTag("url").addText("http://localhost:" + solrServer.getPort() + "/solr").gotoParent()
+            .addTag("url").addText(driver.getRealUrl("SOLR")).gotoParent()
             .gotoRoot();
         return xml;
     }
@@ -264,7 +186,7 @@ public class KnoxRangerTest {
         }
         Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/webhdfs-liststatus-test.json");
 
-        hdfsServer
+        driver.getMock("WEBHDFS")
         .expect()
           .method( "GET" )
           .pathInfo( "/v1/hdfstest" )
@@ -280,7 +202,7 @@ public class KnoxRangerTest {
           .header("X-XSRF-Header", "jksdhfkhdsf")
           .queryParam( "op", "LISTSTATUS" )
         .when()
-          .get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/webhdfs"
+ "/v1/hdfstest" )
+          .get( driver.getUrl("WEBHDFS") + "/v1/hdfstest" )
         .then()
           .statusCode(statusCode)
           .log().body();
@@ -297,7 +219,7 @@ public class KnoxRangerTest {
         }
         Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/cluster-configuration.json");
 
-        stormServer
+        driver.getMock("STORM")
             .expect()
             .method("GET")
             .pathInfo("/api/v1/cluster/configuration")
@@ -310,7 +232,7 @@ public class KnoxRangerTest {
             .auth().preemptive().basic(user, password)
             .header("X-XSRF-Header", "jksdhfkhdsf")
             .header("Accept", "application/json")
-            .when().get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/storm"
+ "/api/v1/cluster/configuration")
+            .when().get( driver.getUrl("STORM") + "/api/v1/cluster/configuration")
             .then()
             .log().all()
             .statusCode(statusCode);
@@ -325,7 +247,7 @@ public class KnoxRangerTest {
         Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/webhbase-table-list.xml");
 
 
-        hbaseServer
+        driver.getMock("WEBHBASE")
         .expect()
         .method( "GET" )
         .pathInfo( "/" )
@@ -340,7 +262,7 @@ public class KnoxRangerTest {
             .auth().preemptive().basic( user, password )
             .header("X-XSRF-Header", "jksdhfkhdsf")
             .header( "Accept", ContentType.XML.toString() )
-            .when().get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/hbase"
)
+            .when().get( driver.getUrl("WEBHBASE") )
             .then()
             .statusCode( statusCode )
             .log().body();
@@ -348,7 +270,7 @@ public class KnoxRangerTest {
 
     private void makeKafkaInvocation(int statusCode, String user, String password) throws
IOException {
 
-        kafkaServer
+        driver.getMock("KAFKA")
         .expect()
         .method( "GET" )
         .pathInfo( "/topics" )
@@ -360,7 +282,7 @@ public class KnoxRangerTest {
             .auth().preemptive().basic( user, password )
             .header("X-XSRF-Header", "jksdhfkhdsf")
         .when()
-            .get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/kafka"
+ "/topics" )
+            .get( driver.getUrl("KAFKA") + "/topics" )
         .then()
             .statusCode(statusCode)
             .log().body();
@@ -374,10 +296,10 @@ public class KnoxRangerTest {
         }
         Path path = FileSystems.getDefault().getPath(basedir, "/src/test/resources/query_response.xml");
 
-        solrServer
+        driver.getMock("SOLR")
         .expect()
         .method("GET")
-        .pathInfo("/solr/gettingstarted/select")
+        .pathInfo("/gettingstarted/select")
         .queryParam("q", "author_s:William+Shakespeare")
         .respond()
         .status(HttpStatus.SC_OK)
@@ -388,11 +310,11 @@ public class KnoxRangerTest {
         .auth().preemptive().basic(user, password)
         .header("X-XSRF-Header", "jksdhfkhdsf")
         .header("Accept", "application/json")
-        .when().get( "http://localhost:" + gateway.getAddresses()[0].getPort() + "/gateway/cluster/solr"
+        .when().get( driver.getUrl("SOLR")
             + "/gettingstarted/select?q=author_s:William+Shakespeare")
         .then()
         .log().all()
         .statusCode(statusCode);
 
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 550580e..69e042f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -183,7 +183,7 @@
         <junit.version>4.12</junit.version>
         <kafka.version>0.10.0.0</kafka.version>
         <kerby.version>1.0.0</kerby.version>
-        <knox.gateway.version>0.13.0</knox.gateway.version>
+        <knox.gateway.version>1.0.0</knox.gateway.version>
         <libpam4j.version>1.8</libpam4j.version>
         <local.lib.dir>${project.basedir}/../lib/local</local.lib.dir>
         <log4j.version>1.2.17</log4j.version>

http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java
----------------------------------------------------------------------
diff --git a/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java
b/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java
index 89f8ed2..a5a0865 100644
--- a/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java
+++ b/ranger-knox-plugin-shim/src/main/java/org/apache/ranger/authorization/knox/deploy/RangerPDPKnoxDeploymentContributor.java
@@ -22,12 +22,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
-import org.apache.hadoop.gateway.deploy.DeploymentContext;
-import org.apache.hadoop.gateway.deploy.ProviderDeploymentContributorBase;
-import org.apache.hadoop.gateway.descriptor.FilterParamDescriptor;
-import org.apache.hadoop.gateway.descriptor.ResourceDescriptor;
-import org.apache.hadoop.gateway.topology.Provider;
-import org.apache.hadoop.gateway.topology.Service;
+import org.apache.knox.gateway.deploy.DeploymentContext;
+import org.apache.knox.gateway.deploy.ProviderDeploymentContributorBase;
+import org.apache.knox.gateway.descriptor.FilterParamDescriptor;
+import org.apache.knox.gateway.descriptor.ResourceDescriptor;
+import org.apache.knox.gateway.topology.Provider;
+import org.apache.knox.gateway.topology.Service;
 
 public class RangerPDPKnoxDeploymentContributor extends ProviderDeploymentContributorBase
{
 

http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ProviderDeploymentContributor
----------------------------------------------------------------------
diff --git a/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ProviderDeploymentContributor
b/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ProviderDeploymentContributor
deleted file mode 100644
index c0c4576..0000000
--- a/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ProviderDeploymentContributor
+++ /dev/null
@@ -1,18 +0,0 @@
-##########################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-##########################################################################
-org.apache.ranger.authorization.knox.deploy.RangerPDPKnoxDeploymentContributor

http://git-wip-us.apache.org/repos/asf/ranger/blob/2de803d1/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ProviderDeploymentContributor
----------------------------------------------------------------------
diff --git a/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ProviderDeploymentContributor
b/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ProviderDeploymentContributor
new file mode 100644
index 0000000..c0c4576
--- /dev/null
+++ b/ranger-knox-plugin-shim/src/main/resources/META-INF/services/org.apache.knox.gateway.deploy.ProviderDeploymentContributor
@@ -0,0 +1,18 @@
+##########################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##########################################################################
+org.apache.ranger.authorization.knox.deploy.RangerPDPKnoxDeploymentContributor


Mime
View raw message