myriad-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dar...@apache.org
Subject incubator-myriad git commit: [Myriad 171] - updateFile had parameter change between Hadoop 2.7.0->2.7.1 [Myriad 190] - Tests fail on Hadoop 2.6.2+
Date Sat, 02 Apr 2016 11:18:55 GMT
Repository: incubator-myriad
Updated Branches:
  refs/heads/master e1db4d8b8 -> 1daad6324


[Myriad 171] - updateFile had parameter change between Hadoop 2.7.0->2.7.1
[Myriad 190] - Tests fail on Hadoop 2.6.2+

Corrects behavior for hadoop 2.7.1+ and hadoop 2.6.2+.

Added support to specify hadoopVer from command line as follows:
```
./gradlew -PhadoopVer=X.Y.Z build
```
where `X.Y.Z` is the hadoop version.

JIRA:
  [Myriad-171] https://issues.apache.org/jira/browse/MYRIAD-171
  [Myriad-190] https://issues.apache.org/jira/browse/MYRIAD-190

Pull Request:
  Closes #63

Author:
  darinj darinj@apache.org


Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/1daad632
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/1daad632
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/1daad632

Branch: refs/heads/master
Commit: 1daad6324ce8b74df0f144600785c1db3d7348d9
Parents: e1db4d8
Author: darinj <darinj.work@gmail.com>
Authored: Mon Mar 14 02:04:26 2016 -0400
Committer: darinj <darinj.work@gmail.com>
Committed: Sat Apr 2 06:59:57 2016 -0400

----------------------------------------------------------------------
 build.gradle                                    |  5 ++-
 gradle.properties                               |  1 -
 .../recovery/MyriadFileSystemRMStateStore.java  | 37 ++++++++++++++++++--
 .../myriad/scheduler/fgs/FGSTestBaseSpec.groovy |  2 ++
 4 files changed, 41 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 918f84c..067a52d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -68,7 +68,10 @@ subprojects {
 
     ext {
         mesosVer = "0.24.1"
-        hadoopVer = "2.7.0"
+        //Allows passing -PhadoopVer=2.6.0 from command line
+        if(!project.hasProperty('hadoopVer')) {
+            hadoopVer = "2.7.0"
+        }
         metricsVer = "3.1.0"
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/gradle.properties
----------------------------------------------------------------------
diff --git a/gradle.properties b/gradle.properties
index 9667034..bca1f77 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -18,6 +18,5 @@
 org.gradle.jvmargs=-Xmx1024m -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8
 org.gradle.parallel=true
 org.gradle.daemon=true
-
 # faster builds:  gradle build -x findBugsM
 

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
index dda9b28..923e29d 100644
--- a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
+++ b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
@@ -20,6 +20,9 @@
 package org.apache.hadoop.yarn.server.resourcemanager.recovery;
 
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileStatus;
@@ -43,11 +46,28 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore
impleme
   private Path myriadPathRoot = null;
   private byte[] myriadStateBytes = null;
 
+  private Method updateFileMethod = null; //This is a cache Method so we do fewer reflection
calls
+
   @Override
   public synchronized void initInternal(Configuration conf) throws Exception {
     super.initInternal(conf);
     Path rootPath = new Path(fsWorkingPath, ROOT_NAME);
     myriadPathRoot = new Path(rootPath, RM_MYRIAD_ROOT);
+    updateFileMethod = getUpdateFileMethod();
+    if (updateFileMethod == null) {
+      //something is broken
+      throw new RuntimeException("Could not find valid updateFile Method");
+    }
+  }
+
+  private Method getUpdateFileMethod() {
+    Method[] methods = super.getClass().getSuperclass().getDeclaredMethods();
+    for (Method m : methods) {
+      if (m.getName().equals("updateFile")) {
+        return m;
+      }
+    }
+    return null;
   }
 
   @Override
@@ -59,7 +79,6 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore
impleme
   @Override
   public synchronized RMState loadState() throws Exception {
     RMState rmState = super.loadState();
-
     Path myriadStatePath = new Path(myriadPathRoot, MYRIAD_STATE_FILE);
     LOGGER.info("Loading state information for Myriad from: " + myriadStatePath);
 
@@ -92,12 +111,26 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore
impleme
 
     LOGGER.debug("Storing state information for Myriad at: " + myriadStatePath);
     try {
-      updateFile(myriadStatePath, sc.toSerializedContext().toByteArray());
+      reflectedUpdateFile(myriadStatePath, sc.toSerializedContext().toByteArray());
     } catch (Exception e) {
       LOGGER.error("State information for Myriad could not be stored at: " + myriadStatePath,
e);
     }
   }
 
+
+
+  protected void reflectedUpdateFile(Path outputPath, byte[] data) throws InvocationTargetException,
IllegalAccessException {
+    Class [] parameters = updateFileMethod.getParameterTypes();
+    if (parameters.length == 2 && parameters[0].equals(Path.class) && parameters[1].isArray())
{
+      updateFileMethod.invoke(this, outputPath, data);
+    } else if (parameters.length == 3 && parameters[0].equals(Path.class) &&
parameters[1].isArray() && parameters[2].isPrimitive()) {
+      updateFileMethod.invoke(this, outputPath, data, true);
+    } else {
+      //something is broken
+      throw new RuntimeException("updateFile Method has unexpected parameters");
+    }
+  }
+
   @Override
   public synchronized void removeMyriadState() throws Exception {
     if (fs.exists(myriadPathRoot)) {

http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
index c769999..b87c1be 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
@@ -21,6 +21,7 @@ package org.apache.myriad.scheduler.fgs
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
 import org.apache.hadoop.yarn.api.records.*
+import org.apache.hadoop.yarn.conf.YarnConfiguration
 import org.apache.hadoop.yarn.event.Dispatcher
 import org.apache.hadoop.yarn.event.EventHandler
 import org.apache.hadoop.yarn.server.resourcemanager.MockNodes
@@ -108,6 +109,7 @@ class FGSTestBaseSpec extends Specification {
         getRMApplicationHistoryWriter() >> writer
         getSystemMetricsPublisher() >> publisher
         getRMNodes() >> rmNodes
+        getYarnConfiguration() >> new YarnConfiguration()
     }
 
     /******************* Offers Related ****************/


Mime
View raw message