phoenix-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject phoenix git commit: PHOENIX-4405 Switch to using Hadoop3
Date Mon, 27 Nov 2017 23:16:56 GMT
Repository: phoenix
Updated Branches:
  refs/heads/5.x-HBase-2.0 309ef0b1a -> 8e5b4131b


PHOENIX-4405 Switch to using Hadoop3

Includes a temporary fix to work around a jetty dependency conflict
between Hadoop and HBase


Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/8e5b4131
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/8e5b4131
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/8e5b4131

Branch: refs/heads/5.x-HBase-2.0
Commit: 8e5b4131b4b9bbbf9aab43bf304504e8238027f0
Parents: 309ef0b
Author: Josh Elser <elserj@apache.org>
Authored: Mon Nov 27 15:25:42 2017 -0500
Committer: Josh Elser <elserj@apache.org>
Committed: Mon Nov 27 15:55:40 2017 -0500

----------------------------------------------------------------------
 phoenix-core/pom.xml                            | 21 ++++++++
 .../phoenix/trace/PhoenixMetricsSink.java       |  2 +-
 .../recovery/TestPerRegionIndexWriteCache.java  | 31 ++++++++++--
 .../org/apache/phoenix/metrics/LoggingSink.java |  2 +-
 phoenix-spark/pom.xml                           |  8 +--
 phoenix-tracing-webapp/pom.xml                  | 16 ++++--
 pom.xml                                         | 52 ++++++++++++++------
 7 files changed, 104 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 3124d9b..0d5affe 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -418,6 +418,27 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <!-- TODO remove after HBASE-19256 -->
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-http</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-security</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-server</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-webapp</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-server</artifactId>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
index fea6d61..65071f0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/PhoenixMetricsSink.java
@@ -34,7 +34,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 
-import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.configuration2.SubsetConfiguration;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseConfiguration;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
index 2bfbcbf..c51ac99 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java
@@ -43,11 +43,16 @@ import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.FSUtils;
 import org.apache.hadoop.hbase.wal.WAL;
 import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestName;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
@@ -60,6 +65,8 @@ public class TestPerRegionIndexWriteCache {
   private static final byte[] qual = Bytes.toBytes("qual");
   private static final byte[] val = Bytes.toBytes("val");
 
+  private static MiniDFSCluster miniDfs = null;
+
   Put p = new Put(row);
   Put p2 = new Put(Bytes.toBytes("other row"));
   {
@@ -71,18 +78,34 @@ public class TestPerRegionIndexWriteCache {
   HRegion r2; // FIXME: Uses private type
   WAL wal;
 
+  @Rule
+  public TestName testName = new TestName();
+
+  @BeforeClass
+  public static void startDfs() throws Exception {
+      miniDfs = TEST_UTIL.startMiniDFSCluster(1);
+  }
+
+  @AfterClass
+  public static void stopDfs() throws Exception {
+      if (miniDfs != null) {
+          miniDfs.shutdown();
+          miniDfs = null;
+      }
+  }
+
   @SuppressWarnings("deprecation")
-@Before
+  @Before
   public void setUp() throws Exception {
-      Path hbaseRootDir = TEST_UTIL.getDataTestDir();
+      Path hbaseRootDir = new Path(getClass().getSimpleName() + "_" + testName.getMethodName());
       TEST_UTIL.getConfiguration().set("hbase.rootdir", hbaseRootDir.toString());
 
-      FileSystem newFS = FileSystem.newInstance(TEST_UTIL.getConfiguration());
+      FileSystem newFS = miniDfs.getFileSystem();
       RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(null).setEndKey(null).setSplit(false).build();
       Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
       Random rn = new Random();
       tableName = TableName.valueOf("TestPerRegion" + rn.nextInt());
-      WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestPerRegionIndexWriteCache");
+      WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), null, getClass().getSimpleName());
       wal = walFactory.getWAL(Bytes.toBytes("logs"), null);
         TableDescriptor htd =
                 TableDescriptorBuilder

http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java b/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java
index 2cea684..1af01ab 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/metrics/LoggingSink.java
@@ -17,7 +17,7 @@
  */
 package org.apache.phoenix.metrics;
 
-import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.configuration2.SubsetConfiguration;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.metrics2.AbstractMetric;

http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-spark/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml
index f3bd9c2..d6ad4d5 100644
--- a/phoenix-spark/pom.xml
+++ b/phoenix-spark/pom.xml
@@ -102,7 +102,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
-      <version>${hadoop-two.version}</version>
+      <version>${hadoop.version}</version>
       <exclusions>
         <exclusion>
           <groupId>log4j</groupId>
@@ -134,7 +134,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
-      <version>${hadoop-two.version}</version>
+      <version>${hadoop.version}</version>
       <exclusions>
         <exclusion>
           <groupId>log4j</groupId>
@@ -166,7 +166,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
-      <version>${hadoop-two.version}</version>
+      <version>${hadoop.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
       <exclusions>
@@ -200,7 +200,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
-      <version>${hadoop-two.version}</version>
+      <version>${hadoop.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
       <exclusions>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/phoenix-tracing-webapp/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-tracing-webapp/pom.xml b/phoenix-tracing-webapp/pom.xml
index 6cc9960..1cb047f 100755
--- a/phoenix-tracing-webapp/pom.xml
+++ b/phoenix-tracing-webapp/pom.xml
@@ -41,15 +41,23 @@
     <dependencies>
       <dependency>
         <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-http</artifactId>
+      </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-security</artifactId>
+      </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
         <artifactId>jetty-server</artifactId>
-        <!-- TODO : pick the same jetty version throughout the project (PHOENIX-2211)-->
-        <version>${jettyVersion}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-servlet</artifactId>
       </dependency>
       <dependency>
         <groupId>org.eclipse.jetty</groupId>
         <artifactId>jetty-webapp</artifactId>
-        <!-- TODO : pick the same jetty version throughout the project (PHOENIX-2211)-->
-        <version>${jettyVersion}</version>
       </dependency>
       <dependency>
           <groupId>org.apache.hadoop</groupId>

http://git-wip-us.apache.org/repos/asf/phoenix/blob/8e5b4131/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bef97db..ac5cd08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -67,12 +67,11 @@
 
     <!-- Hadoop Versions -->
     <hbase.version>2.0.0-beta-1-SNAPSHOT</hbase.version>
-    <hadoop-two.version>2.7.1</hadoop-two.version>
+    <hadoop.version>3.0.0-beta1</hadoop.version>
 
     <!-- Dependency versions -->
     <commons-cli.version>1.2</commons-cli.version>
     <hive.version>1.2.1</hive.version>
-    <hadoop.version>2.7.1</hadoop.version>
     <pig.version>0.13.0</pig.version>
     <jackson.version>1.9.2</jackson.version>
     <antlr.version>3.5.2</antlr.version>
@@ -99,7 +98,7 @@
     <jodatime.version>1.6</jodatime.version>
     <joni.version>2.1.2</joni.version>
     <avatica.version>1.10.0</avatica.version>
-    <jettyVersion>8.1.7.v20120910</jettyVersion>
+    <jetty.version>9.3.19.v20170502</jetty.version>
     <tephra.version>0.13.0-incubating</tephra.version>
     <spark.version>2.0.2</spark.version>
     <scala.version>2.11.8</scala.version>
@@ -655,7 +654,7 @@
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-common</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
         <exclusions>
           <exclusion>
             <groupId>org.xerial.snappy</groupId>
@@ -666,17 +665,17 @@
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-annotations</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-mapreduce-client-core</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-minicluster</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
         <optional>true</optional>
         <scope>test</scope>
       </dependency>
@@ -685,39 +684,39 @@
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-auth</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-mapreduce-client-common</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-client</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-hdfs</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-hdfs</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
         <type>test-jar</type> <!-- this does not work which is typical for
maven.-->
         <scope>test</scope>
       </dependency>
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-minikdc</artifactId>
-        <version>${hadoop-two.version}</version>
+        <version>${hadoop.version}</version>
       </dependency>
 
       <!-- General Dependencies -->
@@ -945,6 +944,31 @@
         <artifactId>i18n-util</artifactId>
         <version>${i18n-util.version}</version>
       </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-http</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-security</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-server</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-servlet</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.eclipse.jetty</groupId>
+        <artifactId>jetty-webapp</artifactId>
+        <version>${jetty.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 


Mime
View raw message