From commits-return-111796-archive-asf-public=cust-asf.ponee.io@lucene.apache.org Thu Dec 12 03:23:14 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 53435180638 for ; Thu, 12 Dec 2019 04:23:14 +0100 (CET) Received: (qmail 59851 invoked by uid 500); 12 Dec 2019 03:23:12 -0000 Mailing-List: contact commits-help@lucene.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@lucene.apache.org Delivered-To: mailing list commits@lucene.apache.org Received: (qmail 59791 invoked by uid 99); 12 Dec 2019 03:23:10 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Dec 2019 03:23:10 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 3CAA08D80D; Thu, 12 Dec 2019 03:23:09 +0000 (UTC) Date: Thu, 12 Dec 2019 03:23:11 +0000 To: "commits@lucene.apache.org" Subject: [lucene-solr] 02/02: SOLR-14053: Fix HDFS tests to be ignored if Native I/O isn't available (remove tests.disableHdfs) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit From: krisden@apache.org In-Reply-To: <157612098952.19629.9119330607156145383@gitbox.apache.org> References: <157612098952.19629.9119330607156145383@gitbox.apache.org> X-Git-Host: gitbox.apache.org X-Git-Repo: lucene-solr X-Git-Refname: refs/heads/branch_8x X-Git-Reftype: branch X-Git-Rev: 5b4266db7d74d8e9fc16d73b14d8306af2ec5caa X-Git-NotificationType: diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated Message-Id: <20191212032310.3CAA08D80D@gitbox.apache.org> This is an automated email from the ASF dual-hosted git repository. krisden pushed a commit to branch branch_8x in repository https://gitbox.apache.org/repos/asf/lucene-solr.git commit 5b4266db7d74d8e9fc16d73b14d8306af2ec5caa Author: Kevin Risden AuthorDate: Wed Dec 11 21:38:25 2019 -0500 SOLR-14053: Fix HDFS tests to be ignored if Native I/O isn't available (remove tests.disableHdfs) Signed-off-by: Kevin Risden --- .../TestSolrCloudWithSecureImpersonation.java | 74 +++++++++++----------- .../org/apache/solr/cloud/hdfs/HdfsTestUtil.java | 23 +++++-- .../hadoop/TestDelegationWithHadoopAuth.java | 9 +-- .../hadoop/TestImpersonationWithHadoopAuth.java | 4 +- .../hadoop/TestSolrCloudWithHadoopAuthPlugin.java | 4 +- .../security/hadoop/TestZkAclsWithHadoopAuth.java | 26 ++------ 6 files changed, 67 insertions(+), 73 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java index c8e4457..a62de35 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java @@ -24,7 +24,6 @@ import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.hadoop.conf.Configuration; -import org.apache.lucene.util.Constants; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -32,8 +31,10 @@ import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.cloud.hdfs.HdfsTestUtil; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.IOUtils; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.request.SolrQueryRequest; @@ -57,23 +58,21 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 { private static String getUsersFirstGroup() throws Exception { String group = "*"; // accept any group if a group can't be found - if (!Constants.WINDOWS) { // does not work on Windows! - org.apache.hadoop.security.Groups hGroups = - new org.apache.hadoop.security.Groups(new Configuration()); - try { - List g = hGroups.getGroups(System.getProperty("user.name")); - if (g != null && g.size() > 0) { - group = g.get(0); - } - } catch (NullPointerException npe) { - // if user/group doesn't exist on test box + org.apache.hadoop.security.Groups hGroups = + new org.apache.hadoop.security.Groups(new Configuration()); + try { + List g = hGroups.getGroups(System.getProperty("user.name")); + if (g != null && g.size() > 0) { + group = g.get(0); } + } catch (NullPointerException npe) { + // if user/group doesn't exist on test box } return group; } private static Map getImpersonatorSettings() throws Exception { - Map filterProps = new TreeMap(); + Map filterProps = new TreeMap<>(); filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "noGroups.hosts", "*"); filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.groups", "*"); filterProps.put(KerberosPlugin.IMPERSONATOR_PREFIX + "anyHostAnyUser.hosts", "*"); @@ -93,7 +92,7 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 { @BeforeClass public static void startup() throws Exception { - assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS); + HdfsTestUtil.checkAssumptions(); System.setProperty("authenticationPlugin", HttpParamDelegationTokenPlugin.class.getName()); System.setProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED, "true"); @@ -142,32 +141,35 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 { } @Before - public void clearCalledIndicator() throws Exception { + public void clearCalledIndicator() { ImpersonatorCollectionsHandler.called.set(false); } @AfterClass public static void shutdown() throws Exception { - if (miniCluster != null) { - miniCluster.shutdown(); - } - miniCluster = null; if (solrClient != null) { - solrClient.close(); - } - solrClient = null; - System.clearProperty("authenticationPlugin"); - System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED); - System.clearProperty("solr.kerberos.cookie.domain"); - Map impSettings = getImpersonatorSettings(); - for (Map.Entry entry : impSettings.entrySet()) { - System.clearProperty(entry.getKey()); + IOUtils.closeQuietly(solrClient); + solrClient = null; } - System.clearProperty("solr.test.sys.prop1"); - System.clearProperty("solr.test.sys.prop2"); - System.clearProperty("collectionsHandler"); + try { + if (miniCluster != null) { + miniCluster.shutdown(); + } + } finally { + miniCluster = null; + System.clearProperty("authenticationPlugin"); + System.clearProperty(KerberosPlugin.DELEGATION_TOKEN_ENABLED); + System.clearProperty("solr.kerberos.cookie.domain"); + Map impSettings = getImpersonatorSettings(); + for (Map.Entry entry : impSettings.entrySet()) { + System.clearProperty(entry.getKey()); + } + System.clearProperty("solr.test.sys.prop1"); + System.clearProperty("solr.test.sys.prop2"); + System.clearProperty("collectionsHandler"); - SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(false); + SolrRequestParsers.DEFAULT.setAddRequestHeadersToContext(false); + } } private void create1ShardCollection(String name, String config, MiniSolrCloudCluster solrCluster) throws Exception { @@ -290,8 +292,7 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 { public void testProxyInvalidRemote() throws Exception { HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class, () -> { - String invalidIpAddress = DEAD_HOST_2; - solrClient.request(getProxyRequest("localHostAnyGroup","bar", "[ff01::114]", invalidIpAddress)); + solrClient.request(getProxyRequest("localHostAnyGroup","bar", "[ff01::114]", DEAD_HOST_2)); }); assertTrue(e.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup"))); } @@ -319,15 +320,12 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 { // try a command to each node, one of them must be forwarded for (JettySolrRunner jetty : miniCluster.getJettySolrRunners()) { - HttpSolrClient client = - new HttpSolrClient.Builder(jetty.getBaseUrl().toString() + "/" + collectionName).build(); - try { + try (HttpSolrClient client = new HttpSolrClient.Builder( + jetty.getBaseUrl().toString() + "/" + collectionName).build()) { ModifiableSolrParams params = new ModifiableSolrParams(); params.set("q", "*:*"); params.set(USER_PARAM, "user"); client.query(params); - } finally { - client.close(); } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java index f6f6588..c86db8b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java +++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HdfsTestUtil.java @@ -39,13 +39,13 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.lucene.util.Constants; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.util.HdfsUtil; -import org.junit.AssumptionViolatedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,6 +73,20 @@ public class HdfsTestUtil { return setupClass(dir, haTesting, true); } + public static void checkAssumptions() { + checkHadoopWindows(); + checkFastDateFormat(); + checkGeneratedIdMatches(); + } + + /** + * Hadoop integration tests fail on Windows without Hadoop NativeIO + */ + private static void checkHadoopWindows() { + LuceneTestCase.assumeTrue("Hadoop does not work on Windows without Hadoop NativeIO", + !Constants.WINDOWS || NativeIO.isAvailable()); + } + /** * Checks that commons-lang3 FastDateFormat works with configured locale */ @@ -96,8 +110,7 @@ public class HdfsTestUtil { } public static MiniDFSCluster setupClass(String dir, boolean safeModeTesting, boolean haTesting) throws Exception { - checkFastDateFormat(); - checkGeneratedIdMatches(); + checkAssumptions(); if (!HA_TESTING_ENABLED) haTesting = false; @@ -132,10 +145,6 @@ public class HdfsTestUtil { dfsClusterBuilder.nnTopology(MiniDFSNNTopology.simpleHATopology()); } - if (!NativeIO.isAvailable()) { - throw new AssumptionViolatedException("NativeIO not available for HDFS."); - } - MiniDFSCluster dfsCluster = dfsClusterBuilder.build(); HdfsUtil.TEST_CONF = getClientConfiguration(dfsCluster); System.setProperty("solr.hdfs.home", getDataDir(dfsCluster, "solr_hdfs_home")); diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java index b87b854..500f8ad 100644 --- a/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java +++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java @@ -24,7 +24,6 @@ import java.util.Set; import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; import org.apache.hadoop.util.Time; import org.apache.http.HttpStatus; -import org.apache.lucene.util.Constants; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.embedded.JettySolrRunner; @@ -35,10 +34,12 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.DelegationTokenRequest; import org.apache.solr.client.solrj.response.DelegationTokenResponse; import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.cloud.hdfs.HdfsTestUtil; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.IOUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -51,7 +52,7 @@ public class TestDelegationWithHadoopAuth extends SolrCloudTestCase { @BeforeClass public static void setupClass() throws Exception { - assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS); + HdfsTestUtil.checkAssumptions(); configureCluster(NUM_SERVERS)// nodes .withSecurityJson(TEST_PATH().resolve("security").resolve("hadoop_simple_auth_with_delegation.json")) @@ -70,12 +71,12 @@ public class TestDelegationWithHadoopAuth extends SolrCloudTestCase { @AfterClass public static void tearDownClass() throws Exception { if (primarySolrClient != null) { - primarySolrClient.close(); + IOUtils.closeQuietly(primarySolrClient); primarySolrClient = null; } if (secondarySolrClient != null) { - secondarySolrClient.close(); + IOUtils.closeQuietly(secondarySolrClient); secondarySolrClient = null; } } diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java index d603fd2..8cdcbf8 100644 --- a/solr/core/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java +++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestImpersonationWithHadoopAuth.java @@ -23,12 +23,12 @@ import java.nio.file.Path; import java.util.HashMap; import java.util.Map; -import org.apache.lucene.util.Constants; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.cloud.hdfs.HdfsTestUtil; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.Utils; import org.apache.solr.security.HadoopAuthPlugin; @@ -50,7 +50,7 @@ public class TestImpersonationWithHadoopAuth extends SolrCloudTestCase { @SuppressWarnings("unchecked") @BeforeClass public static void setupClass() throws Exception { - assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS); + HdfsTestUtil.checkAssumptions(); InetAddress loopback = InetAddress.getLoopbackAddress(); Path securityJsonPath = TEST_PATH().resolve("security").resolve("hadoop_simple_auth_with_delegation.json"); diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java index ae323a2..3043394 100644 --- a/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java +++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestSolrCloudWithHadoopAuthPlugin.java @@ -20,7 +20,6 @@ import java.io.File; import java.nio.charset.StandardCharsets; import org.apache.commons.io.FileUtils; -import org.apache.lucene.util.Constants; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; @@ -28,6 +27,7 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.AbstractDistribZkTestBase; import org.apache.solr.cloud.KerberosTestServices; import org.apache.solr.cloud.SolrCloudAuthTestCase; +import org.apache.solr.cloud.hdfs.HdfsTestUtil; import org.apache.solr.common.SolrInputDocument; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -41,7 +41,7 @@ public class TestSolrCloudWithHadoopAuthPlugin extends SolrCloudAuthTestCase { @BeforeClass public static void setupClass() throws Exception { - assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS); + HdfsTestUtil.checkAssumptions(); setupMiniKdc(); diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java index 1cd0ae6..8846652 100644 --- a/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java +++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestZkAclsWithHadoopAuth.java @@ -27,16 +27,14 @@ import java.util.List; import java.util.Locale; import java.util.concurrent.TimeUnit; -import org.apache.lucene.util.Constants; import org.apache.solr.cloud.MiniSolrCloudCluster; import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.cloud.hdfs.HdfsTestUtil; import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.VMParamsAllAndReadonlyDigestZkACLProvider; import org.apache.solr.common.cloud.VMParamsSingleSetCredentialsDigestZkCredentialsProvider; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.ACL; @@ -58,7 +56,7 @@ public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase { @BeforeClass public static void setupClass() throws Exception { - assumeFalse("Hadoop does not work on Windows", Constants.WINDOWS); + HdfsTestUtil.checkAssumptions(); System.setProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, VMParamsAllAndReadonlyDigestZkACLProvider.class.getName()); @@ -88,16 +86,9 @@ public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase { @Test public void testZkAcls() throws Exception { - ZooKeeper keeper = null; - try { - keeper = new ZooKeeper(cluster.getZkServer().getZkAddress(), (int) TimeUnit.MINUTES.toMillis(1), new Watcher() { - @Override - public void process(WatchedEvent arg0) { - // Do nothing - } - }); - - keeper.addAuthInfo("digest", ("solr:"+SOLR_PASSWD).getBytes(StandardCharsets.UTF_8)); + try (ZooKeeper keeper = new ZooKeeper(cluster.getZkServer().getZkAddress(), + (int) TimeUnit.MINUTES.toMillis(1), arg0 -> {/* Do nothing */})) { + keeper.addAuthInfo("digest", ("solr:" + SOLR_PASSWD).getBytes(StandardCharsets.UTF_8)); // Test well known paths. checkNonSecurityACLs(keeper, "/solr.xml"); @@ -106,13 +97,8 @@ public class TestZkAclsWithHadoopAuth extends SolrCloudTestCase { // Now test all ZK tree. String zkHost = cluster.getSolrClient().getZkHost(); - String zkChroot = zkHost.contains("/")? zkHost.substring(zkHost.indexOf("/")): null; + String zkChroot = zkHost.contains("/") ? zkHost.substring(zkHost.indexOf("/")) : null; walkZkTree(keeper, zkChroot, "/"); - - } finally { - if (keeper != null) { - keeper.close(); - } } }