hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1590272 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/coprocessor/endpoints/ main/java/org/apache/hadoop/hbase/thrift/ main/java/org/apache/hadoop/hbase/util/ test/java/org/apache/hadoop/hbase/thrift/
Date Sat, 26 Apr 2014 18:18:32 GMT
Author: liyin
Date: Sat Apr 26 18:18:32 2014
New Revision: 1590272

URL: http://svn.apache.org/r1590272
Log:
[master] Change of CuttingClassLoader and ThrfitResultSerializer/2 for loading different version
of Thrift

Author: daviddeng

Summary:
`instance` in `ThrfitResultSerializer2` is declared as Object and used with reflection
`protocolClass` is loaded with same classloader of `ThrfitResultSerializer`
Add some logs

Test Plan: `TestThrfitResultSerializer`

Reviewers: rshroff, manukranthk, liyintang, gauravm, adela

Reviewed By: adela

CC: hbase-eng@, ssl-diffs@, arice

Differential Revision: https://phabricator.fb.com/D1294940

Task ID: 3547474

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java?rev=1590272&r1=1590271&r2=1590272&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/coprocessor/endpoints/HTableEndpointClient.java
Sat Apr 26 18:18:32 2014
@@ -104,7 +104,7 @@ public class HTableEndpointClient implem
         results.put(region, caller.call(ep));
       }
     } catch (UndeclaredThrowableException e) {
-      ExceptionUtils.throwIOExcetion(e.getUndeclaredThrowable());
+      throw ExceptionUtils.toIOException(e.getUndeclaredThrowable());
     }
 
     return results;

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java?rev=1590272&r1=1590271&r2=1590272&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/CuttingClassLoader.java
Sat Apr 26 18:18:32 2014
@@ -21,32 +21,63 @@ package org.apache.hadoop.hbase.thrift;
 
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 /**
  * The classloader allows the searching firstly in some path, if not found
  * search in parent.
+ *
+ * The class is useful for loading classes whose dependency is incompatible with
+ * the current classloader.
+ *
+ * skipClasses are set to make those classes to be loaded with parent loader
+ * and thus can be used by classes loaded by both classloaders. They are
+ * commonly classes used in the interface. If the class doesn't exist in the
+ * cutting path, it is not necessary to put in skipClasses.
+ *
+ * Class loaded with different classloader is not assignable to each other, use
+ * reflection to call its methods.
  */
 public class CuttingClassLoader extends URLClassLoader {
-  public CuttingClassLoader(URL[] urls, ClassLoader parent) {
+  private static Log LOG = LogFactory.getLog(CuttingClassLoader.class);
+  private Set<String> skipClasses = new HashSet<>();
+
+  public CuttingClassLoader(URL[] urls, ClassLoader parent,
+      String... skipClasses) {
     super(urls, parent);
+    Collections.addAll(this.skipClasses, skipClasses);
   }
 
   @Override
   protected synchronized Class<?> loadClass(String name, boolean resolve)
       throws ClassNotFoundException {
+    if (skipClasses.contains(name)) {
+      // Find in parent
+      LOG.debug("Finding " + name + " in parent path!");
+      return super.loadClass(name, resolve);
+    }
+
     // Find in loaded classes
     Class<?> clazz = findLoadedClass(name);
     if (clazz != null) {
+      LOG.debug("Found " + name + " in loaded class!");
       return clazz;
     }
 
     try {
       // Find in cutting paths
+      LOG.debug("Finding " + name + " in cutting path!");
       return findClass(name);
     } catch (ClassNotFoundException e) {
     }
 
     // Find in parent
+    LOG.debug("Finding " + name + " in parent path!");
     return super.loadClass(name, resolve);
   }
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java?rev=1590272&r1=1590271&r2=1590272&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer.java
Sat Apr 26 18:18:32 2014
@@ -28,8 +28,8 @@ import org.apache.hadoop.conf.Configurab
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
-import org.apache.hadoop.hbase.mapred.TableInputFormat;
 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
+import org.apache.hadoop.hbase.util.ExceptionUtils;
 import org.apache.hadoop.io.serializer.Serializer;
 import org.apache.thrift.TException;
 import org.apache.thrift.protocol.TCompactProtocol;
@@ -38,18 +38,18 @@ import org.apache.thrift.transport.TIOSt
 import org.apache.thrift.transport.TTransport;
 
 /**
- * A serializer for use with {@link TableInputFormat}. Serializes results as TResult.
+ * A serializer for use with TableInputFormat. Serializes results as TResult.
  */
 public class ThriftResultSerializer implements Serializer<Object>, Configurable {
 
   private static final Log LOG = LogFactory.getLog(ThriftResultSerializer.class);
 
-  public static final String PROTOCOL_CONF_KEY = "hbase.thrift.result.serializer.protocol.class";
-  
+  public static final String PROTOCOL_CONF_KEY =
+      "hbase.thrift.result.serializer.protocol.class";
+
   private OutputStream out;
-  
-  @SuppressWarnings("rawtypes")
-  private Class protocolClass = TCompactProtocol.class;
+
+  private Class<? extends TProtocol> protocolClass = TCompactProtocol.class;
 
   private TProtocol prot;
   private DataOutput dataOut;
@@ -63,15 +63,15 @@ public class ThriftResultSerializer impl
     transport = new TIOStreamTransport(out);
 
     LOG.info("Using Thrift protocol: " + protocolClass.getName());
-    
+
     try {
-      Constructor<TProtocol> constructor =
-          protocolClass.getConstructor(new Class[] { TTransport.class });
+      Constructor<? extends TProtocol> constructor =
+          protocolClass.getConstructor(TTransport.class);
       prot = constructor.newInstance(transport);
     } catch (Exception ex) {
-      throw new RuntimeException(ex);
+      throw ExceptionUtils.toIOException(ex);
     }
-    
+
     if (out instanceof DataOutput) {
       dataOut = (DataOutput) out;
     } else {
@@ -79,10 +79,9 @@ public class ThriftResultSerializer impl
     }
   }
 
-  @SuppressWarnings("rawtypes")
   @Override
   public void serialize(Object t) throws IOException {
-    Class klass = t.getClass();
+    Class<?> klass = t.getClass();
     if (klass == Result.class) {
       Result result = (Result) t;
       TRowResult tResult = ThriftUtilities.oneRowResult(result);
@@ -105,10 +104,21 @@ public class ThriftResultSerializer impl
     out.close();
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public void setConf(Configuration conf) {
     this.conf = conf;
-    protocolClass = conf.getClass(PROTOCOL_CONF_KEY, protocolClass);
+    String protoclCLassName =
+        conf.get(PROTOCOL_CONF_KEY, protocolClass.getName());
+    try {
+      protocolClass =
+          (Class<? extends TProtocol>) Class.forName(protoclCLassName);
+    } catch (ClassNotFoundException e) {
+      throw new RuntimeException(e);
+    }
+
+    LOG.info("Classloader of " + protocolClass.getName() + " is "
+        + protocolClass.getClassLoader());
   }
 
   @Override

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java?rev=1590272&r1=1590271&r2=1590272&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/thrift/ThriftResultSerializer2.java
Sat Apr 26 18:18:32 2014
@@ -22,12 +22,20 @@ package org.apache.hadoop.hbase.thrift;
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.lang.reflect.Method;
 import java.net.URL;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.hadoop.hbase.util.ExceptionUtils;
 import org.apache.hadoop.io.serializer.Serializer;
 
+
 /**
  * A wrapper class which loads <code>ThriftResultSerializer</code> class with
a
  * <code>CuttingClassLoader</code>.
@@ -37,16 +45,25 @@ import org.apache.hadoop.io.serializer.S
  */
 public class ThriftResultSerializer2 implements Serializer<Object>,
     Configurable {
-  ThriftResultSerializer instance;
+  private static Log LOG = LogFactory.getLog(ThriftResultSerializer2.class);
+  // ThriftResultSerializer instance;
+  Object instance;
   /**
    * Key to the configuration of classpath to cut
    */
   public static final String CUTTING_CLASSPATH_KEY =
       "hbase.thrift.result.serializer.cutting.classpath";
 
+  private Method getConfMethod;
+  private Method openMethod;
+  private Method serializeMethod;
+  private Method closeMethod;
+
+  @SuppressWarnings("resource")
   @Override
   public void setConf(Configuration conf) {
     String classPath = conf.get(CUTTING_CLASSPATH_KEY, "");
+    LOG.info(CUTTING_CLASSPATH_KEY + " is set to " + classPath);
     try {
       // make the URL array of cutting paths
       String[] parts = classPath.split(File.pathSeparator);
@@ -56,37 +73,69 @@ public class ThriftResultSerializer2 imp
       }
 
       // Create the ClassLoader
+      // Skip all classes in the interface so that both side can use it.
       ClassLoader classLoader = new CuttingClassLoader(urls,
- this.getClass().getClassLoader());
+              ThriftResultSerializer.class.getClassLoader(),
+              Result.class.getName(), ImmutableBytesWritable.class.getName(),
+              KeyValue.class.getName(), Configuration.class.getName());
       // and load the class.
-      Class<?> cls = Class.forName(ThriftResultSerializer.class.getName(),
-          true, classLoader);
+      Class<?> cls =
+          classLoader.loadClass(ThriftResultSerializer.class.getName());
       // Then create the new instance.
-      instance = (ThriftResultSerializer) cls.newInstance();
+      instance = cls.newInstance();
     } catch (Exception e) {
-      throw new RuntimeException(e);
+      throw ExceptionUtils.toRuntimeException(e);
     }
 
-    instance.setConf(conf);
+    try {
+      getConfMethod = instance.getClass().getMethod("getConf");
+      openMethod = instance.getClass().getMethod("open", OutputStream.class);
+      serializeMethod =
+          instance.getClass().getMethod("serialize", Object.class);
+      closeMethod = instance.getClass().getMethod("close");
+
+      Method setConfMethod =
+          instance.getClass().getMethod("setConf", Configuration.class);
+      setConfMethod.invoke(instance, conf);
+    } catch (Exception e) {
+      throw ExceptionUtils.toRuntimeException(e);
+    }
   }
 
   @Override
   public Configuration getConf() {
-    return instance.getConf();
+    try {
+      return (Configuration) getConfMethod.invoke(instance);
+    } catch (Exception e) {
+      throw ExceptionUtils.toRuntimeException(e);
+    }
   }
 
   @Override
   public void open(OutputStream out) throws IOException {
-    instance.open(out);
+    try {
+      openMethod.invoke(instance, out);
+    } catch (Exception e) {
+      throw ExceptionUtils.toRuntimeException(e);
+    }
   }
 
   @Override
   public void serialize(Object t) throws IOException {
-    instance.serialize(t);
+    try {
+      serializeMethod.invoke(instance, t);
+    } catch (Exception e) {
+      throw ExceptionUtils.toRuntimeException(e);
+    }
   }
 
   @Override
   public void close() throws IOException {
-    instance.close();
+    try {
+      closeMethod.invoke(instance);
+    } catch (Exception e) {
+      throw ExceptionUtils.toRuntimeException(e);
+    }
   }
+
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java?rev=1590272&r1=1590271&r2=1590272&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/util/ExceptionUtils.java
Sat Apr 26 18:18:32 2014
@@ -20,25 +20,70 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.UndeclaredThrowableException;
 
 /**
  * Some utility methods for handling exception.
  */
 public class ExceptionUtils {
   /**
-   * Throws an arbitrary exception to an IOException. Wrap the exception with
-   * an IOException if the exception cannot be thrown directly.
+   * If t is an RuntimeException throw it directly, or
+   * If t is an InvocationTargetException, recursively call toIOException
+   * with target exception, or
+   * If t is an UndeclaredThrowableException, recursively call toIOException
+   * with undeclared throwable, or
+   * Otherwise convert it into an IOException. Wrap the exception with an
+   * IOException if necessary.
+   *
+   * DO NOT use it if you want to wrap all exception including RuntimeException
+   * into an IOException.
    */
-  public static void throwIOExcetion(Throwable t)
-      throws IOException {
+  public static IOException toIOException(Throwable t) {
+    if (t instanceof InvocationTargetException) {
+      return toIOException(((InvocationTargetException) t)
+          .getTargetException());
+    }
+
+    if (t instanceof UndeclaredThrowableException) {
+      return toIOException(((UndeclaredThrowableException) t)
+          .getUndeclaredThrowable());
+    }
+
     if (t instanceof IOException) {
-      throw (IOException) t;
+      return (IOException) t;
     }
 
     if (t instanceof RuntimeException) {
       throw (RuntimeException) t;
     }
 
-    throw new IOException(t);
+    return new IOException(t);
+  }
+
+  /**
+   * If t is an RuntimeException return it directly, or
+   * If t is an InvocationTargetException, recursively call toRuntimeException
+   * with target exception, or
+   * If t is an UndeclaredThrowableException, recursively call
+   * toRuntimeException with undeclared throwable, or
+   * Otherwise wrap it into an toRuntimeException.
+   */
+  public static RuntimeException toRuntimeException(Throwable t) {
+    if (t instanceof InvocationTargetException) {
+      return toRuntimeException(((InvocationTargetException) t)
+          .getTargetException());
+    }
+
+    if (t instanceof UndeclaredThrowableException) {
+      return toRuntimeException(((UndeclaredThrowableException) t)
+          .getUndeclaredThrowable());
+    }
+
+    if (t instanceof RuntimeException) {
+      return (RuntimeException) t;
+    }
+
+    return new RuntimeException(t);
   }
 }

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java?rev=1590272&r1=1590271&r2=1590272&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java
(original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftResultSerializer.java
Sat Apr 26 18:18:32 2014
@@ -20,9 +20,6 @@
 package org.apache.hadoop.hbase.thrift;
 
 import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -38,15 +35,15 @@ import org.junit.Test;
  */
 public class TestThriftResultSerializer {
 
-  private KeyValue[] constuctKvList(int n) {
-    List<KeyValue> list = new ArrayList<KeyValue>();
+  private static KeyValue[] constuctKvList(int n) {
+    KeyValue[] kvs = new KeyValue[n];
     for (int i = 0; i < n; i++) {
       KeyValue kv = new KeyValue(Bytes.toBytes("myRow" + i),
           Bytes.toBytes("myCF"), Bytes.toBytes("myQualifier"), 12345L,
           Bytes.toBytes("myValue"));
-      list.add(kv);
+      kvs[i] = kv;
     }
-    return list.toArray(new KeyValue[list.size()]);
+    return kvs;
   }
 
   @Test
@@ -69,28 +66,19 @@ public class TestThriftResultSerializer 
     Assert.assertTrue("output is empty", out.size() > 0);
   }
 
-  private static String findThriftPath() {
-    String[] paths = System.getProperty("java.class.path").split(
-        File.pathSeparator);
-    for (String path : paths) {
-      if (path.contains("/libthrift-")) {
-        return path;
-      }
-    }
-    return "";
-  }
-
   @Test
   public void testWithClassPath() throws Exception {
     ByteArrayOutputStream out = new ByteArrayOutputStream();
 
     ThriftResultSerializer2 trs = new ThriftResultSerializer2();
 
-    String cp = findThriftPath();
+    String cp = System.getProperty("java.class.path");
     System.out.println("Cutting classpath: " + cp);
 
     Configuration conf = HBaseConfiguration.create();
     conf.set(ThriftResultSerializer2.CUTTING_CLASSPATH_KEY, cp);
+    conf.set(ThriftResultSerializer.PROTOCOL_CONF_KEY,
+        "org.apache.thrift.protocol.TCompactProtocol");
     trs.setConf(conf);
 
     trs.open(out);
@@ -103,4 +91,33 @@ public class TestThriftResultSerializer 
 
     Assert.assertTrue("output is empty", out.size() > 0);
   }
+
+  public static void main(String[] args) throws Exception {
+    System.out.println("java.class.path: "
+        + System.getProperty("java.class.path"));
+
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    ThriftResultSerializer2 trs = new ThriftResultSerializer2();
+
+    Configuration conf = HBaseConfiguration.create();
+    if (args.length >= 1) {
+      System.out.println("Setting "
+          + ThriftResultSerializer2.CUTTING_CLASSPATH_KEY + " to " + args[0]);
+      conf.set(ThriftResultSerializer2.CUTTING_CLASSPATH_KEY, args[0]);
+    }
+    if (args.length >= 2) {
+      System.out.println("Setting " + ThriftResultSerializer.PROTOCOL_CONF_KEY
+          + " to " + args[1]);
+      conf.set(ThriftResultSerializer.PROTOCOL_CONF_KEY, args[1]);
+    }
+    trs.setConf(conf);
+
+    trs.open(out);
+    try {
+      trs.serialize(new ImmutableBytesWritable(Bytes.toBytes("row")));
+      trs.serialize(new Result(constuctKvList(10)));
+    } finally {
+      trs.close();
+    }
+  }
 }



Mime
View raw message