hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From la...@apache.org
Subject hbase git commit: HBASE-15059 Allow 0.94 to compile against Hadoop 2.7.x.
Date Tue, 05 Jan 2016 05:36:06 GMT
Repository: hbase
Updated Branches:
  refs/heads/0.94 30b0c9824 -> dd5660444


HBASE-15059 Allow 0.94 to compile against Hadoop 2.7.x.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dd566044
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dd566044
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dd566044

Branch: refs/heads/0.94
Commit: dd5660444a2587840bf982559d5b48f8a8e74e3c
Parents: 30b0c98
Author: Lars Hofhansl <larsh@apache.org>
Authored: Mon Jan 4 21:36:19 2016 -0800
Committer: Lars Hofhansl <larsh@apache.org>
Committed: Mon Jan 4 21:36:19 2016 -0800

----------------------------------------------------------------------
 dev-support/build-proto.sh                      |  37 +++++++
 pom.xml                                         |  98 ++++++++++++++++
 .../hadoop/hbase/master/MasterDumpServlet.java  |   5 +-
 .../metrics/file/TimeStampingFileContext.java   | 111 -------------------
 .../hbase/regionserver/RSDumpServlet.java       |   5 +-
 .../org/apache/hadoop/hbase/util/Threads.java   |  49 +++++++-
 .../file/TestTimeStampingMetricsContext.java    | 107 ------------------
 7 files changed, 191 insertions(+), 221 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/dd566044/dev-support/build-proto.sh
----------------------------------------------------------------------
diff --git a/dev-support/build-proto.sh b/dev-support/build-proto.sh
new file mode 100644
index 0000000..96380f8
--- /dev/null
+++ b/dev-support/build-proto.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+##
+# script to run protoc to generate protocol buf files.
+# usage: ./build-proto.sh
+#
+
+which protoc
+if [ $? != 0 ] ; then
+  echo "Must have protoc compiler in your path to generate code"
+  exit 1
+fi
+
+HBASE_DIR=`dirname $0`/..
+PROTO_DIR=$HBASE_DIR/src/main/protobuf
+JAVA_DIR=$HBASE_DIR/src/main/java
+
+set -x
+for f in $PROTO_DIR/*.proto ; do
+  protoc -I$PROTO_DIR --java_out=$JAVA_DIR $f
+done
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/dd566044/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 19c7ed5..6507012 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2385,6 +2385,104 @@
       </build>
     </profile>
 
+    <!--
+      profile for building against Hadoop 2.7. Activate using:
+       mvn -Dhadoop.profile=2.7
+
+      *** NOTE:
+       You MUST ensure that the protobuf Java classes are generated with protoc 2.5.0.
+       Do that by calling  dev-support/build-proto.sh  with protoc 2.5.0 in your path
+       before building HBase; that will regenerate the Java classes.
+       (HBase will not work with Hadoop 2.6+ with the older classes generated with protoc
2.4.1)
+
+       The necessary runtime dependencies for protobuf and commons.io are pulled in automatically.
+
+      Example:
+       $ protoc - -version
+       check that the output indicates 2.5.0
+
+       $ dev-support/build-proto.sh
+       $ mvn -Dhadoop.profile=2.7 ...
+    -->
+    <profile>
+      <id>hadoop-2.7</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>2.7</value>
+        </property>
+      </activation>
+      <properties>
+        <hadoop.version>2.7.1</hadoop.version>
+        <slf4j.version>1.6.1</slf4j.version>
+        <protobuf.version>2.5.0</protobuf.version>
+        <commons-io.version>2.4</commons-io.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-annotations</artifactId>
+          <version>${hadoop.version}</version>
+        </dependency>
+        <!-- test deps for hadoop-2.0 profile -->
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-minicluster</artifactId>
+          <version>${hadoop.version}</version>
+          <scope>compile</scope>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>add-test-resource</id>
+                <goals>
+                  <goal>add-test-resource</goal>
+                </goals>
+                <configuration>
+                  <resources>
+                    <resource>
+                      <directory>src/test/resources</directory>
+                      <includes>
+                        <include>hbase-site.xml</include>
+                      </includes>
+                    </resource>
+                  </resources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+          <plugin>
+            <artifactId>maven-dependency-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>create-mrapp-generated-classpath</id>
+                <phase>generate-test-resources</phase>
+                <goals>
+                  <goal>build-classpath</goal>
+                </goals>
+                <configuration>
+                  <!-- needed to run the unit test for DS to generate
+                  the required classpath that is required in the env
+                  of the launch container in the mini mr/yarn cluster
+                  -->
+                  <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
 
     <!-- profiles for the tests
          See as well the properties of the project for the values

http://git-wip-us.apache.org/repos/asf/hbase/blob/dd566044/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
index 876eda4..608979a 100644
--- a/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/master/MasterDumpServlet.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.master;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.util.Date;
 import java.util.Map;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
 import org.apache.hadoop.hbase.monitoring.LogMonitoring;
 import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.util.ReflectionUtils;
 
 public class MasterDumpServlet extends StateDumpServlet {
@@ -79,7 +81,8 @@ public class MasterDumpServlet extends StateDumpServlet {
     
     out.println("\n\nStacks:");
     out.println(LINE);
-    ReflectionUtils.printThreadInfo(out, "");
+    PrintStream ps = new PrintStream(os, false, "UTF-8");
+    Threads.printThreadInfo(ps, "");
     
     out.println("\n\nMaster configuration:");
     out.println(LINE);

http://git-wip-us.apache.org/repos/asf/hbase/blob/dd566044/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java
b/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java
deleted file mode 100644
index 000e0d3..0000000
--- a/src/main/java/org/apache/hadoop/hbase/metrics/file/TimeStampingFileContext.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Copyright 2008 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.metrics.file;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.hadoop.metrics.ContextFactory;
-import org.apache.hadoop.metrics.file.FileContext;
-import org.apache.hadoop.metrics.spi.OutputRecord;
-
-/**
- * Add timestamp to {@link org.apache.hadoop.metrics.file.FileContext#emitRecord(String,
String, OutputRecord)}.
- */
-public class TimeStampingFileContext extends FileContext {
-  // Copies bunch of FileContext here because writer and file are private in
-  // superclass.
-  private File file = null;
-  private PrintWriter writer = null;
-  private final SimpleDateFormat sdf;
-
-  public TimeStampingFileContext() {
-    super();
-    this.sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
-  }
-
-  @Override
-  public void init(String contextName, ContextFactory factory) {
-    super.init(contextName, factory);
-    String fileName = getAttribute(FILE_NAME_PROPERTY);
-    if (fileName != null) {
-      file = new File(fileName);
-    }
-  }
-
-  @Override
-  public void startMonitoring() throws IOException {
-    if (file == null) {
-      writer = new PrintWriter(new BufferedOutputStream(System.out));
-    } else {
-      writer = new PrintWriter(new FileWriter(file, true));
-    }
-    super.startMonitoring();
-  }
-
-  @Override
-  public void stopMonitoring() {
-    super.stopMonitoring();
-    if (writer != null) {
-      writer.close();
-      writer = null;
-    }
-  }
-
-  private synchronized String iso8601() {
-    return this.sdf.format(new Date());
-  }
-
-  @Override
-  public void emitRecord(String contextName, String recordName,
-      OutputRecord outRec) {
-    writer.print(iso8601());
-    writer.print(" ");
-    writer.print(contextName);
-    writer.print(".");
-    writer.print(recordName);
-    String separator = ": ";
-    for (String tagName : outRec.getTagNames()) {
-      writer.print(separator);
-      separator = ", ";
-      writer.print(tagName);
-      writer.print("=");
-      writer.print(outRec.getTag(tagName));
-    }
-    for (String metricName : outRec.getMetricNames()) {
-      writer.print(separator);
-      separator = ", ";
-      writer.print(metricName);
-      writer.print("=");
-      writer.print(outRec.getMetric(metricName));
-    }
-    writer.println();
-  }
-
-  @Override
-  public void flush() {
-    writer.flush();
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hbase/blob/dd566044/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
index 979cd52..2df824b 100644
--- a/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
+++ b/src/main/java/org/apache/hadoop/hbase/regionserver/RSDumpServlet.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.util.Date;
 
@@ -31,6 +32,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.monitoring.LogMonitoring;
 import org.apache.hadoop.hbase.monitoring.StateDumpServlet;
 import org.apache.hadoop.hbase.monitoring.TaskMonitor;
+import org.apache.hadoop.hbase.util.Threads;
 import org.apache.hadoop.util.ReflectionUtils;
 
 public class RSDumpServlet extends StateDumpServlet {
@@ -77,7 +79,8 @@ public class RSDumpServlet extends StateDumpServlet {
     
     out.println("\n\nStacks:");
     out.println(LINE);
-    ReflectionUtils.printThreadInfo(out, "");
+    PrintStream ps = new PrintStream(os, false, "UTF-8");
+    Threads.printThreadInfo(ps, "");
     
     out.println("\n\nRS Configuration:");
     out.println(LINE);

http://git-wip-us.apache.org/repos/asf/hbase/blob/dd566044/src/main/java/org/apache/hadoop/hbase/util/Threads.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/hadoop/hbase/util/Threads.java b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
index a5a4d13..5a51bcc 100644
--- a/src/main/java/org/apache/hadoop/hbase/util/Threads.java
+++ b/src/main/java/org/apache/hadoop/hbase/util/Threads.java
@@ -19,8 +19,11 @@
  */
 package org.apache.hadoop.hbase.util;
 
+import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.lang.Thread.UncaughtExceptionHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -116,7 +119,7 @@ public class Threads {
     while (t.isAlive()) {
       t.join(60 * 1000);
       if (t.isAlive()) {
-        ReflectionUtils.printThreadInfo(new PrintWriter(System.out),
+        printThreadInfo(System.out,
             "Automatic Stack Trace every 60 seconds waiting on " +
             t.getName());
       }
@@ -229,4 +232,48 @@ public class Threads {
 
     };
   }
+
+  private static Method printThreadInfoMethod = null;
+  private static boolean printThreadInfoMethodWithPrintStream = true;
+
+  /**
+   * Print all of the thread's information and stack traces. Wrapper around Hadoop's method.
+   *
+   * @param stream the stream to
+   * @param title a string title for the stack trace
+   */
+  public static void printThreadInfo(PrintStream stream, String title) {
+
+    if (printThreadInfoMethod == null) {
+      try {
+        // Hadoop 2.7+ declares printThreadInfo(PrintStream, String)
+        printThreadInfoMethod = ReflectionUtils.class.getMethod("printThreadInfo",
+          PrintStream.class, String.class);
+      } catch (NoSuchMethodException e) {
+        // Hadoop 2.6 and earlier declares printThreadInfo(PrintWriter, String)
+        printThreadInfoMethodWithPrintStream = false;
+        try {
+          printThreadInfoMethod = ReflectionUtils.class.getMethod("printThreadInfo",
+            PrintWriter.class, String.class);
+        } catch (NoSuchMethodException e1) {
+          throw new RuntimeException("Cannot find method. Check hadoop jars linked", e1);
+        }
+      }
+      printThreadInfoMethod.setAccessible(true);
+    }
+
+    try {
+      if (printThreadInfoMethodWithPrintStream) {
+        printThreadInfoMethod.invoke(null, stream, title);
+      } else {
+        printThreadInfoMethod.invoke(null, new PrintWriter(stream), title);
+      }
+    } catch (IllegalAccessException e) {
+      throw new RuntimeException(e.getCause());
+    } catch (IllegalArgumentException e) {
+      throw new RuntimeException(e.getCause());
+    } catch (InvocationTargetException e) {
+      throw new RuntimeException(e.getCause());
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/dd566044/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java
b/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java
deleted file mode 100644
index 0700508..0000000
--- a/src/test/java/org/apache/hadoop/hbase/metrics/file/TestTimeStampingMetricsContext.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.metrics.file;
-
-import java.io.File;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Collection;
-import java.util.Date;
-import java.util.Map;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hbase.SmallTests;
-
-import org.junit.*;
-import org.junit.experimental.categories.Category;
-
-import static org.junit.Assert.*;
-
-/**
- * Test for TimeStampingMetricsContext functionality.
- * (FQN class names are used to suppress javac warnings in imports.)
- */
-@Category(SmallTests.class)
-@SuppressWarnings("deprecation")
-public class TestTimeStampingMetricsContext {
-
-  private static final int updatePeriodSeconds = 2;
-
-  private TimeStampingFileContext mc;
-
-  @Test
-  public void testFileUpdate() throws Exception {
-    final Date start = new Date();
-    final File metricOutFile = FileUtil.createLocalTempFile(
-      new File(FileUtils.getTempDirectory(),getClass().getName() + "-out-"), "", true);
-    assertTrue(metricOutFile.exists());
-    assertEquals(0L, metricOutFile.length());
-
-    mc = new TimeStampingFileContext();
-    org.apache.hadoop.metrics.ContextFactory cf
-      = org.apache.hadoop.metrics.ContextFactory.getFactory();
-    cf.setAttribute("test1.fileName", metricOutFile.getAbsolutePath());
-    cf.setAttribute("test1.period", Integer.toString(updatePeriodSeconds));
-    mc.init("test1", cf);
-
-    assertEquals("test1", mc.getContextName());
-
-    org.apache.hadoop.metrics.MetricsRecord r = mc.createRecord("testRecord");
-    r.setTag("testTag1", "testTagValue1");
-    r.setTag("testTag2", "testTagValue2");
-    r.setMetric("testMetric1", 1);
-    r.setMetric("testMetric2", 33);
-    r.update();
-
-    mc.startMonitoring();
-    assertTrue(mc.isMonitoring());
-
-    // wait 3/2 of the update period:
-    Thread.sleep((1000 * updatePeriodSeconds * 3)/2);
-
-    mc.stopMonitoring();
-    assertFalse(mc.isMonitoring());
-
-    mc.close();
-
-    Map<String, Collection<org.apache.hadoop.metrics.spi.OutputRecord>> m = mc.getAllRecords();
-    assertEquals(1, m.size());
-    Collection<org.apache.hadoop.metrics.spi.OutputRecord> outputRecords = m.get("testRecord");
-    assertNotNull(outputRecords);
-    assertEquals(1, outputRecords.size());
-    org.apache.hadoop.metrics.spi.OutputRecord outputRecord = outputRecords.iterator().next();
-    assertNotNull(outputRecord);
-
-    String outStr = FileUtils.readFileToString(metricOutFile);
-    assertTrue(outStr.length() > 0);
-    int pos = outStr.indexOf(" ");
-    String time = outStr.substring(0, pos);
-
-    DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
-    Date date = df.parse(time);
-    assertTrue(date.after(start));
-    assertTrue(date.before(new Date()));
-
-    String reminder = outStr.substring(pos);
-    assertEquals(" test1.testRecord: testTag1=testTagValue1, testTag2=testTagValue2, testMetric1=1,"
-      +" testMetric2=33\n", reminder);
-  }
-
-}


Mime
View raw message