giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ikabi...@apache.org
Subject git commit: updated refs/heads/trunk to ad84a99
Date Fri, 13 Jan 2017 18:49:18 GMT
Repository: giraph
Updated Branches:
  refs/heads/trunk 3b53360ab -> ad84a99e2


GIRAPH-1129

closes #13


Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/ad84a99e
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/ad84a99e
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/ad84a99e

Branch: refs/heads/trunk
Commit: ad84a99e2ba630fa81aa6823515bfdeac1f3fe8a
Parents: 3b53360
Author: Igor Kabiljo <ikabiljo@fb.com>
Authored: Fri Jan 13 10:48:45 2017 -0800
Committer: Igor Kabiljo <ikabiljo@fb.com>
Committed: Fri Jan 13 10:48:45 2017 -0800

----------------------------------------------------------------------
 giraph-block-app-8/pom.xml                      |   4 +
 .../giraph/compiling/ClassCodeGenerator.java    |  35 +++++
 .../giraph/compiling/FactoryCodeGenerator.java  | 115 ++++++++++++++
 .../giraph/compiling/LambdaConfOption.java      | 103 +++++++++++++
 .../compiling/ObjectFactoryConfOption.java      | 134 ++++++++++++++++
 .../compiling/ObjectInitializerConfOption.java  |  39 +++++
 .../giraph/compiling/RuntimeClassGenerator.java | 113 ++++++++++++++
 .../RuntimeObjectFactoryGenerator.java          |  68 +++++++++
 .../apache/giraph/compiling/package-info.java   |  21 +++
 .../giraph/compiling/TestLambdaConfOption.java  | 109 +++++++++++++
 .../TestRuntimeObjectFactoryCreationUtils.java  | 151 +++++++++++++++++++
 pom.xml                                         |  12 +-
 12 files changed, 901 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/pom.xml b/giraph-block-app-8/pom.xml
index 82885be..96035e7 100644
--- a/giraph-block-app-8/pom.xml
+++ b/giraph-block-app-8/pom.xml
@@ -119,6 +119,10 @@ under the License.
       <groupId>log4j</groupId>
       <artifactId>log4j</artifactId>
     </dependency>
+    <dependency>
+      <groupId>net.openhft</groupId>
+      <artifactId>compiler</artifactId>
+    </dependency>
 
     <!-- runtime dependency -->
     <dependency>

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ClassCodeGenerator.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ClassCodeGenerator.java
b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ClassCodeGenerator.java
new file mode 100644
index 0000000..a2dcaee
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ClassCodeGenerator.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+/**
+ * Interface which can generate code from code snippet
+ */
+public interface ClassCodeGenerator {
+  /**
+   * Generate code for creating a class inside of specified package with given
+   * class name, using provided code snippet
+   *
+   * @param packageName Package to create class in
+   * @param className Name of the class
+   * @param codeSnippet Cod snippet for this class
+   * @return Code used to generate the class
+   */
+  String generateCode(
+      String packageName, String className, String codeSnippet);
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/FactoryCodeGenerator.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/FactoryCodeGenerator.java
b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/FactoryCodeGenerator.java
new file mode 100644
index 0000000..dab03d3
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/FactoryCodeGenerator.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+
+import org.apache.giraph.utils.Factory;
+
+import com.google.common.reflect.TypeToken;
+
+/**
+ * Class which creates object factories from code snippet to create and return
+ * object
+ */
+public class FactoryCodeGenerator implements ClassCodeGenerator {
+  private final Type objectType;
+
+  public FactoryCodeGenerator(Type objectType) {
+    this.objectType = objectType;
+  }
+
+  @Override
+  public String generateCode(
+      String packageName, String className, String creationCode) {
+    StringBuilder src = new StringBuilder();
+    src.append("package ").append(packageName).append(";\n");
+    // Import Factory
+    src.append("import ").append(Factory.class.getName()).append(";\n");
+
+    appendTypeImport(src, objectType);
+    String simpleName = getSimpleName(objectType);
+
+    // Allow classes from java.util to be used
+    src.append("import java.util.*;\n");
+    // Object compiled in one VM cannot be deserialized in other,
+    // so mark generated class as NonKryoWritable
+    src.append(
+        "import org.apache.giraph.writable.kryo.markers.NonKryoWritable;\n");
+    src.append("public class ").append(className).append(
+        " implements ").append(Factory.class.getSimpleName()).append(
+            "<").append(simpleName).append(">, NonKryoWritable {\n");
+    src.append("public ").append(simpleName).append(" create() {\n");
+    src.append(creationCode).append("\n").append("}\n").append("}\n");
+    return src.toString();
+  }
+
+  public static String getSimpleName(TypeToken<?> typeToken) {
+    return getSimpleName(typeToken.getType());
+  }
+
+  public static String getSimpleName(Type type) {
+    return (type instanceof Class) ?
+        ((Class<?>) type).getCanonicalName() :
+        type.toString();
+  }
+
+  private static void appendTypeImport(StringBuilder src, Type objectType) {
+    if (objectType == null) {
+      return;
+    }
+
+    if (objectType instanceof Class) {
+      appendClassImport(src, (Class<?>) objectType);
+    } else if (objectType instanceof ParameterizedType) {
+      appendTypeImport(src, ((ParameterizedType) objectType).getRawType());
+      appendTypeImport(src, ((ParameterizedType) objectType).getOwnerType());
+
+      for (Type innerType :
+            ((ParameterizedType) objectType).getActualTypeArguments()) {
+        appendTypeImport(src, innerType);
+      }
+    } else {
+      throw new IllegalArgumentException(
+          "Unsupported " + objectType + " of type " + objectType.getClass());
+    }
+  }
+
+  private static void appendClassImport(
+      StringBuilder src, Class<?> objectClass) {
+    // Allow classes from the same package to be used
+    src.append("import ").append(
+        objectClass.getPackage().getName()).append(".*;\n");
+    // If objectClass is inner class then above import is not enough
+    src.append("import ").append(objectClass.getCanonicalName()).append(";\n");
+  }
+
+  @Override
+  public int hashCode() {
+    return objectType.hashCode();
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (obj == null || !(obj instanceof FactoryCodeGenerator)) {
+      return false;
+    }
+    return objectType.equals(((FactoryCodeGenerator) obj).objectType);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/LambdaConfOption.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/LambdaConfOption.java
b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/LambdaConfOption.java
new file mode 100644
index 0000000..84e3407
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/LambdaConfOption.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import static org.apache.giraph.compiling.FactoryCodeGenerator.getSimpleName;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Type;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import com.google.common.base.Preconditions;
+import com.google.common.reflect.TypeToken;
+
+/**
+ * Conf option which is configured by specifying lambda body, and it returns
+ * compiled instance of wanted functional interface with given body.
+ * See TestLambdaConfOption for example.
+ *
+ * @param <T> Functional interface type
+ */
+public class LambdaConfOption<T> extends ObjectFactoryConfOption<T> {
+  private final String[] argumentNames;
+
+  public LambdaConfOption(String key, Class<T> classType, String codeSnippet,
+      String description, String... argumentNames) {
+    super(key, classType, codeSnippet, description);
+    this.argumentNames = argumentNames;
+  }
+
+  public LambdaConfOption(
+      String key, TypeToken<T> interfaceTypeToken, String codeSnippet,
+      String description, String... arguments) {
+    super(key, interfaceTypeToken, codeSnippet, description);
+    this.argumentNames = arguments;
+  }
+
+  @Override
+  protected String createFullCodeSnippet(String codeSnippet) {
+    if (codeSnippet == null || codeSnippet.isEmpty()) {
+      return null;
+    }
+
+    return createLambdaCode(codeSnippet, getInterfaceType(), argumentNames);
+  }
+
+  public static String createLambdaCode(
+      String body, Type type, String[] argumentNames) {
+    TypeToken<?> typeToken = TypeToken.of(type);
+    Class<?> rawType = typeToken.getRawType();
+
+    if (rawType.isInterface()) {
+      return  "return (" +
+          Stream.of(argumentNames).collect(Collectors.joining(", ")) +
+          ") -> " + body + ";";
+    } else {
+      // Long workaround for abstract classes
+      Method sam = null;
+
+      for (Method m : rawType.getMethods()) {
+        if (Modifier.isAbstract(m.getModifiers())) {
+          Preconditions.checkState(sam == null);
+          sam = m;
+        }
+      }
+      Preconditions.checkState(sam != null);
+
+      Type[] params = sam.getGenericParameterTypes();
+      Preconditions.checkState(params.length == argumentNames.length);
+
+      String argList = IntStream.range(0, params.length).mapToObj(
+          (i) -> getSimpleName(typeToken.resolveType(params[i])) + " " +
+              argumentNames[i]
+      ).collect(Collectors.joining(", "));
+
+      return "  return new " + getSimpleName(type) + "() {\n" +
+        "    @Override\n" +
+        "    public " +
+        getSimpleName(typeToken.resolveType(sam.getGenericReturnType())) +
+        " " + sam.getName() + "(" + argList + ") {\n" +
+        "      return " + body + ";\n" +
+        "    }\n" +
+        "  };";
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectFactoryConfOption.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectFactoryConfOption.java
b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectFactoryConfOption.java
new file mode 100644
index 0000000..412763b
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectFactoryConfOption.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.lang.reflect.Type;
+
+import org.apache.giraph.conf.StrConfOption;
+import org.apache.giraph.function.Supplier;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.reflect.TypeToken;
+
+/**
+ * Option in configuration which can create objects
+ *
+ * @param <T> Type of objects for this option
+ */
+public class ObjectFactoryConfOption<T> {
+  /** Conf option which holds code snippet for creating objects */
+  private final StrConfOption codeSnippetConfOption;
+  /** Base interface for class */
+  private final Type interfaceType;
+
+  public ObjectFactoryConfOption(
+      String key, Class<T> interfaceClass, String defaultValue,
+      String description) {
+    codeSnippetConfOption = new StrConfOption(key, defaultValue, description);
+    this.interfaceType = interfaceClass;
+  }
+
+  public ObjectFactoryConfOption(
+      String key, TypeToken<T> interfaceTypeToken, String defaultValue,
+      String description) {
+    codeSnippetConfOption = new StrConfOption(key, defaultValue, description);
+    this.interfaceType = interfaceTypeToken.getType();
+  }
+
+  /**
+   * Create object from configuration, or return null if conf option is empty.
+   *
+   * Return objects are NonKryoWritable.
+   *
+   * @param conf Configuration
+   * @return New object, or null if conf option is empty
+   */
+  public final T createObject(Configuration conf) {
+    final String codeSnippet = createFullCodeSnippet(
+        codeSnippetConfOption.get(conf));
+    if (codeSnippet == null || codeSnippet.isEmpty()) {
+      return null;
+    } else {
+      return RuntimeObjectFactoryGenerator.<T>createFactory(
+          codeSnippet, interfaceType).create();
+    }
+  }
+
+  /**
+   * Take user passed codeSnippet and create full code snippet to be given to
+   * RuntimeObjectFactoryGenerator.
+   *
+   * Subclasses can override this method to customize it's logic.
+   */
+  protected String createFullCodeSnippet(String codeSnippet) {
+    return codeSnippet;
+  }
+
+  /**
+   * Create object cached supplier from configuration, which stores snippet for
+   * serializaition, working around objects themselves being NonKryoWritable.
+   *
+   * Object is created only once per new or deserialized instance of returned
+   * supplier.
+   */
+  public final Supplier<T> createSupplier(Configuration conf) {
+    final String codeSnippet = createFullCodeSnippet(
+        codeSnippetConfOption.get(conf));
+    final Type type = this.interfaceType;
+    if (codeSnippet == null || codeSnippet.isEmpty()) {
+      return null;
+    }
+    return createNewSupplier(codeSnippet, type);
+  }
+
+  private static <T> Supplier<T> createNewSupplier(
+      final String codeSnippet, final Type type) {
+    return RuntimeObjectFactoryGenerator.createCachedSupplier(
+        codeSnippet, type);
+  }
+
+  public void setCodeSnippet(Configuration conf, String codeSnippet) {
+    codeSnippetConfOption.set(conf, codeSnippet);
+  }
+
+  public void setCodeSnippetIfUnset(Configuration conf, String codeSnippet) {
+    codeSnippetConfOption.setIfUnset(conf, codeSnippet);
+  }
+
+  public boolean isDefaultValue(Configuration conf) {
+    return codeSnippetConfOption.isDefaultValue(conf);
+  }
+
+  public boolean isEmpty(Configuration conf) {
+    final String codeSnippet = createFullCodeSnippet(
+        codeSnippetConfOption.get(conf));
+    return codeSnippet == null || codeSnippet.isEmpty();
+  }
+
+  public String getCodeSnippet(Configuration conf) {
+    return codeSnippetConfOption.get(conf);
+  }
+
+  public Type getInterfaceType() {
+    return interfaceType;
+  }
+
+  public String getInterfaceName() {
+    return FactoryCodeGenerator.getSimpleName(interfaceType);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectInitializerConfOption.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectInitializerConfOption.java
b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectInitializerConfOption.java
new file mode 100644
index 0000000..9568a59
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectInitializerConfOption.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+/**
+ * Conf option which can initialize an object of specific class with desired
+ * parameters set.
+ * See TestRuntimeObjectFactoryCreationUtils for example.
+ * Make sure fields which you want to set are protected.
+ *
+ * @param <T> Class type
+ */
+public class ObjectInitializerConfOption<T>
+    extends ObjectFactoryConfOption<T> {
+  public ObjectInitializerConfOption(
+      String key, Class<T> classType, String codeSnippet, String description) {
+    super(key, classType, codeSnippet, description);
+  }
+
+  @Override
+  protected String createFullCodeSnippet(String codeSnippet) {
+    return "return new " + getInterfaceName() + "(){{" + codeSnippet + "}};";
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeClassGenerator.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeClassGenerator.java
b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeClassGenerator.java
new file mode 100644
index 0000000..32681f2
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeClassGenerator.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+import net.openhft.compiler.CachedCompiler;
+
+/**
+ * Helper class for creating classes at runtime
+ */
+public class RuntimeClassGenerator {
+  private static final Logger LOG = Logger.getLogger(
+      RuntimeClassGenerator.class);
+
+  /**
+   * In order not to create multiple classes for exactly same Code generator
+   * and String, this keeps all known classes
+   */
+  private static final
+  Map<ClassCodeGenerator, Map<String, Class<?>>> KNOWN_CLASSES =
+    new HashMap<>();
+
+  /** Counter for number of created classes */
+  private static final
+  AtomicInteger CREATED_CLASSES_COUNTER = new AtomicInteger();
+
+  protected RuntimeClassGenerator() {
+  }
+
+  public static synchronized <T> Class<T> getOrCreateClass(
+      ClassCodeGenerator classCodeGenerator, String codeSnippet) {
+    Map<String, Class<?>> knownClassCodeGenerators =
+        KNOWN_CLASSES.get(classCodeGenerator);
+    if (knownClassCodeGenerators == null) {
+      knownClassCodeGenerators = new HashMap<>();
+      KNOWN_CLASSES.put(classCodeGenerator, knownClassCodeGenerators);
+    } else {
+      Class<?> generatedClass = knownClassCodeGenerators.get(codeSnippet);
+      if (generatedClass != null) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(
+              "Reusing class for " + classCodeGenerator + " " + codeSnippet);
+        }
+        return (Class<T>) generatedClass;
+      }
+    }
+    String packageName = classCodeGenerator.getClass().getPackage().getName();
+    String className = classCodeGenerator.getClass().getSimpleName() +
+        CREATED_CLASSES_COUNTER.getAndIncrement();
+    String fullClassName = packageName + "." + className;
+    String classCode = classCodeGenerator.generateCode(
+        packageName, className, codeSnippet);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Creating " + fullClassName + " with \n" + classCode);
+    }
+    Class<T> generatedClass = generateClass(fullClassName, classCode);
+    knownClassCodeGenerators.put(codeSnippet, generatedClass);
+    return generatedClass;
+  }
+
+  /**
+   * Create class with a name, in the package and with source code.
+   */
+  public static synchronized <T> Class<T> createClassOnce(
+      String className, String packageName, String classCode) {
+    return generateClass(packageName + "." + className, classCode);
+  }
+
+  private static <T> Class<T> generateClass(
+      String fullClassName, String classCode) {
+    try {
+      Class<T> generatedClass = new CachedCompiler(null, null).loadFromJava(
+          fullClassName, classCode);
+      // Confirm that class was loaded into default class loader
+      Preconditions.checkState(generatedClass == Class.forName(fullClassName));
+      return generatedClass;
+    } catch (ClassNotFoundException e) {
+      throw new IllegalArgumentException(
+          "Couldn't be compile class " + fullClassName + " , with code: " +
+          classCode,
+          e);
+    }
+  }
+
+  /**
+   * For test
+   */
+  static int getCreatedClassesCount() {
+    return CREATED_CLASSES_COUNTER.get();
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeObjectFactoryGenerator.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeObjectFactoryGenerator.java
b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeObjectFactoryGenerator.java
new file mode 100644
index 0000000..fe14fb0
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeObjectFactoryGenerator.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.lang.reflect.Type;
+
+import org.apache.giraph.function.Supplier;
+import org.apache.giraph.utils.Factory;
+
+/**
+ * Helper class for creating object factories at runtime
+ */
+public class RuntimeObjectFactoryGenerator extends RuntimeClassGenerator {
+  private RuntimeObjectFactoryGenerator() {
+  }
+
+  public static <T> Factory<T> createFactory(
+      String creationCode, Class<T> objectClass) {
+    return createFactory(creationCode, (Type) objectClass);
+  }
+
+  public static <T> Factory<T> createFactory(
+      String creationCode, Type objectType) {
+    FactoryCodeGenerator factoryCodeGenerator =
+        new FactoryCodeGenerator(objectType);
+    Class<Factory<T>> factoryClass = getOrCreateClass(
+        factoryCodeGenerator, creationCode);
+    try {
+      return factoryClass.newInstance();
+    } catch (InstantiationException | IllegalAccessException e) {
+      throw new IllegalStateException(e);
+    }
+  }
+
+  public static <T> Supplier<T> createCachedSupplier(
+      String creationCode, Type objectType) {
+    Supplier<T> supplier = new Supplier<T>() {
+      private transient T cachedValue;
+
+      @Override
+      public T get() {
+        if (cachedValue == null) {
+          cachedValue = RuntimeObjectFactoryGenerator.<T>createFactory(
+              creationCode, objectType).create();
+        }
+        return cachedValue;
+      }
+    };
+    // Compile in current scope, and fail fast if there are compile errors
+    supplier.get();
+    return supplier;
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/package-info.java
b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/package-info.java
new file mode 100644
index 0000000..0b48bf3
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Utilities for compiling code at runtime.
+ */
+package org.apache.giraph.compiling;

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestLambdaConfOption.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestLambdaConfOption.java
b/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestLambdaConfOption.java
new file mode 100644
index 0000000..59f688e
--- /dev/null
+++ b/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestLambdaConfOption.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.io.Serializable;
+
+import org.apache.giraph.function.Function;
+import org.apache.giraph.function.primitive.Obj2LongFunction;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.reflect.TypeToken;
+
+public class TestLambdaConfOption {
+  @Before
+  public void setLogging() {
+    Logger.getLogger(RuntimeClassGenerator.class).setLevel(Level.DEBUG);
+  }
+
+  @Test
+  public void testLambdaConfOptionClass() throws NoSuchFieldException, SecurityException
{
+    Configuration conf = new Configuration();
+    LambdaConfOption<ValueInIteration> testObjectFactoryConfOption =
+        new LambdaConfOption<>("testFactory", ValueInIteration.class, "", "", "iter");
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "");
+    Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "iter + 5");
+    ValueInIteration f = testObjectFactoryConfOption.createObject(conf);
+    Assert.assertEquals(10, f.get(5), 0.0);
+  }
+
+  @Test
+  public void testLambdaConfOptionTypeToken() throws NoSuchFieldException, SecurityException
{
+    Configuration conf = new Configuration();
+    LambdaConfOption<Obj2LongFunction<LongWritable>> testObjectFactoryConfOption
=
+        new LambdaConfOption<>("testFactory", new TypeToken<Obj2LongFunction<LongWritable>>(){},
"", "", "id");
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "");
+    Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "id.get()");
+    Obj2LongFunction<LongWritable> f = testObjectFactoryConfOption.createObject(conf);
+    Assert.assertEquals(10, f.apply(new LongWritable(10)));
+  }
+
+  @Test
+  public void testLambdaConfOptionTypeToken2() throws NoSuchFieldException, SecurityException
{
+    Configuration conf = new Configuration();
+    LambdaConfOption<Function<LongWritable, DoubleWritable>> testObjectFactoryConfOption
=
+        new LambdaConfOption<>("testFactory", new TypeToken<Function<LongWritable,
DoubleWritable>>(){}, "", "", "id");
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "");
+    Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "new DoubleWritable(id.get())");
+    Function<LongWritable, DoubleWritable> f = testObjectFactoryConfOption.createObject(conf);
+    Assert.assertEquals(10, f.apply(new LongWritable(10)).get(), 0.0);
+  }
+
+  /**
+   * Value that depends on current iteration.
+   */
+  public interface ValueInIteration extends Serializable {
+    float get(int iteration);
+  }
+
+  public static abstract class AbstractClass implements ValueInIteration {
+    public static int get5() {
+      return 5;
+    }
+  }
+
+  @Test
+  public void testLambdaConfOptionAbstractClass() throws NoSuchFieldException, SecurityException
{
+    Configuration conf = new Configuration();
+    LambdaConfOption<AbstractClass> testObjectFactoryConfOption =
+        new LambdaConfOption<>("testFactory", AbstractClass.class, "", "", "iter");
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "");
+    Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "iter + get5()");
+    ValueInIteration f = testObjectFactoryConfOption.createObject(conf);
+    Assert.assertEquals(10, f.get(5), 0.0);
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestRuntimeObjectFactoryCreationUtils.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestRuntimeObjectFactoryCreationUtils.java
b/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestRuntimeObjectFactoryCreationUtils.java
new file mode 100644
index 0000000..2be280b
--- /dev/null
+++ b/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestRuntimeObjectFactoryCreationUtils.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestRuntimeObjectFactoryCreationUtils {
+  @Before
+  public void setLogging() {
+    Logger.getLogger(RuntimeClassGenerator.class).setLevel(Level.DEBUG);
+  }
+
+  @Test
+  public void testRuntimeObjectFactoryCreationUtils() {
+    int startCount = RuntimeObjectFactoryGenerator.getCreatedClassesCount();
+
+    TestClass1 testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+        "return new TestClass1(1, \"A\", 0.5);", TestClass1.class).create();
+    Assert.assertEquals(testClass1Object.x, 1);
+    Assert.assertEquals(testClass1Object.y, "A");
+    Assert.assertEquals(testClass1Object.z, 0.5, 0);
+    Assert.assertEquals(startCount + 1,
+        RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+    TestClass2 testClass2Object = RuntimeObjectFactoryGenerator.createFactory(
+        "return new TestClass2(5);", TestClass2.class).create();
+    Assert.assertEquals(testClass2Object.a, 5);
+    Assert.assertEquals(startCount + 2,
+        RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+    testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+        "return new TestClass1(5, \"XY\", 1.5).setZ(3.5);", TestClass1.class).create();
+    Assert.assertEquals(testClass1Object.x, 5);
+    Assert.assertEquals(testClass1Object.y, "XY");
+    Assert.assertEquals(testClass1Object.z, 3.5, 0);
+    Assert.assertEquals(startCount + 3,
+        RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+    testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+        "return new TestClass1(1, \"A\", 0.5);", TestClass1.class).create();
+    Assert.assertEquals(testClass1Object.x, 1);
+    Assert.assertEquals(testClass1Object.y, "A");
+    Assert.assertEquals(testClass1Object.z, 0.5, 0);
+    Assert.assertEquals(startCount + 3,
+        RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+    testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+        "TestClass1 ret = new TestClass1(5, \"XY\", 1.5); ret.setZ(3.5); return ret;",
+        TestClass1.class).create();
+    Assert.assertEquals(testClass1Object.x, 5);
+    Assert.assertEquals(testClass1Object.y, "XY");
+    Assert.assertEquals(testClass1Object.z, 3.5, 0);
+    Assert.assertEquals(startCount + 4,
+        RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+    testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+        "return new TestClass1() {{ x=1;y=\"A\";z=0.5; }};",
+        TestClass1.class).create();
+    Assert.assertEquals(testClass1Object.x, 1);
+    Assert.assertEquals(testClass1Object.y, "A");
+    Assert.assertEquals(testClass1Object.z, 0.5, 0);
+    Assert.assertEquals(startCount + 5,
+        RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+  }
+
+  @Test
+  public void testObjectConfOption() {
+    Configuration conf = new Configuration();
+    ObjectFactoryConfOption<TestClass1> testObjectFactoryConfOption = new ObjectFactoryConfOption<>(
+        "testFactory", TestClass1.class, "return new TestClass1(50, \"XY\", 1.5);", "");
+    TestClass1 testClass1Object = testObjectFactoryConfOption.createObject(conf);
+    Assert.assertEquals(testClass1Object.x, 50);
+    Assert.assertEquals(testClass1Object.y, "XY");
+    Assert.assertEquals(testClass1Object.z, 1.5, 0);
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "");
+    Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+    testObjectFactoryConfOption.setCodeSnippet(conf, "return new TestClass1(10, \"A\", 0.5);");
+    testClass1Object = testObjectFactoryConfOption.createObject(conf);
+    Assert.assertEquals(testClass1Object.x, 10);
+    Assert.assertEquals(testClass1Object.y, "A");
+    Assert.assertEquals(testClass1Object.z, 0.5, 0);
+
+    TestClass1 testClass1OtherObject = testObjectFactoryConfOption.createObject(conf);
+    Assert.assertEquals(testClass1OtherObject.x, 10);
+    Assert.assertEquals(testClass1OtherObject.y, "A");
+    Assert.assertEquals(testClass1OtherObject.z, 0.5, 0);
+
+    // Verify that createObject returns different instance
+    testClass1OtherObject.setZ(5);
+    Assert.assertEquals(testClass1OtherObject.z, 5, 0);
+    Assert.assertEquals(testClass1Object.z, 0.5, 0);
+
+    ObjectInitializerConfOption<TestClass1> testObjectInitializerConfOption =
+        new ObjectInitializerConfOption<>("testInitializer", TestClass1.class,
+            "x=3;y=\"Y\";z=3.5;", "");
+    testClass1Object = testObjectInitializerConfOption.createObject(conf);
+    Assert.assertEquals(testClass1Object.x, 3);
+    Assert.assertEquals(testClass1Object.y, "Y");
+    Assert.assertEquals(testClass1Object.z, 3.5, 0);
+  }
+
+  public static class TestClass1 {
+    protected int x;
+    protected String y;
+    protected double z;
+
+    public TestClass1(int x, String y, double z) {
+      this.x = x;
+      this.y = y;
+      this.z = z;
+    }
+
+    public TestClass1() {
+    }
+
+    public TestClass1 setZ(double z) {
+      this.z = z;
+      return this;
+    }
+  }
+
+  public static class TestClass2 {
+    private final int a;
+
+    public TestClass2(int a) {
+      this.a = a;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 017fd1f..91128f4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -349,11 +349,12 @@ under the License.
     <dep.log4j.version>1.2.17</dep.log4j.version>
     <dep.mockito.version>1.9.5</dep.mockito.version>
     <!-- note: old version of netty is required by hadoop_facebook for tests to succeed
-->
-    <dep.oldnetty.version>3.2.2.Final</dep.oldnetty.version>
     <dep.netty.version>4.0.14.Final</dep.netty.version>
+    <dep.oldnetty.version>3.2.2.Final</dep.oldnetty.version>
     <dep.objenesis.version>2.1</dep.objenesis.version>
+    <dep.openhft-compiler.version>2.2.1</dep.openhft-compiler.version>
     <dep.paranamer.version>2.5.2</dep.paranamer.version>
-    <dep.slf4j.version>1.7.5</dep.slf4j.version>
+    <dep.slf4j.version>1.7.6</dep.slf4j.version>
     <dep.tinkerpop.rexter.version>2.4.0</dep.tinkerpop.rexter.version>
     <dep.typetools.version>0.2.1</dep.typetools.version>
     <dep.yammer-metrics.version>2.2.0</dep.yammer-metrics.version>
@@ -2072,7 +2073,12 @@ under the License.
         <artifactId>slf4j-log4j12</artifactId>
         <version>${dep.slf4j.version}</version>
       </dependency>
-
+      <dependency>
+        <groupId>net.openhft</groupId>
+        <artifactId>compiler</artifactId>
+        <version>${dep.openhft-compiler.version}</version>
+      </dependency>
+      
       <!-- provided dependencies. sorted lexicographically. -->
       <dependency>
         <groupId>commons-collections</groupId>


Mime
View raw message