tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-787. Revert Guava dependency to 11.0.2. (sseth)
Date Tue, 18 Feb 2014 22:47:55 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master 06bc6eabe -> 87f3ea351


TEZ-787. Revert Guava dependency to 11.0.2. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/87f3ea35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/87f3ea35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/87f3ea35

Branch: refs/heads/master
Commit: 87f3ea351db18cde8c50385dde37e69f534515ad
Parents: 06bc6ea
Author: Siddharth Seth <sseth@apache.org>
Authored: Tue Feb 18 14:47:24 2014 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Tue Feb 18 14:47:24 2014 -0800

----------------------------------------------------------------------
 pom.xml                                         |  2 +-
 .../api/events/CompositeDataMovementEvent.java  | 45 +++++++++++++-------
 .../apache/tez/dag/app/rm/TaskScheduler.java    |  5 ++-
 .../common/resources/MemoryDistributor.java     |  5 ++-
 4 files changed, 38 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/87f3ea35/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 30d818a..83b39ad 100644
--- a/pom.xml
+++ b/pom.xml
@@ -297,7 +297,7 @@
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
-        <version>15.0</version>
+        <version>11.0.2</version>
       </dependency>
       <dependency>
         <groupId>org.codehaus.jettison</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/87f3ea35/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
index f7d7878..2abf1ff 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
@@ -18,13 +18,9 @@
 
 package org.apache.tez.runtime.api.events;
 
-import org.apache.tez.runtime.api.Event;
+import java.util.Iterator;
 
-import com.google.common.collect.ContiguousSet;
-import com.google.common.collect.DiscreteDomain;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Range;
-import com.google.common.base.Function;
+import org.apache.tez.runtime.api.Event;
 
 /**
  * A convenience class to specify multiple DataMovementEvents which share the
@@ -83,16 +79,35 @@ public class CompositeDataMovementEvent extends Event {
   }
 
   public Iterable<DataMovementEvent> getEvents() {
-    // Overkill. For now is enough to just run a loop over the ints.
-    Range<Integer> range = Range.closedOpen(sourceIndexStart, sourceIndexEnd);
-    ContiguousSet<Integer> intRange = ContiguousSet.create(range, DiscreteDomain.integers());
-    return Iterables.transform(intRange, new Function<Integer, DataMovementEvent>()
{
-      public DataMovementEvent apply(Integer integer) {
-        DataMovementEvent dmEvent = new DataMovementEvent(integer, userPayload);
-        dmEvent.setVersion(version);
-        return dmEvent;
+
+    return new Iterable<DataMovementEvent>() {
+      
+      @Override
+      public Iterator<DataMovementEvent> iterator() {
+        return new Iterator<DataMovementEvent>() {
+          
+          int currentPos = sourceIndexStart;
+
+          @Override
+          public boolean hasNext() {
+            return currentPos < sourceIndexEnd;
+          }
+
+          @Override
+          public DataMovementEvent next() {
+            DataMovementEvent dmEvent = new DataMovementEvent(currentPos, userPayload);
+            currentPos++;
+            dmEvent.setVersion(version);
+            return dmEvent;
+          }
+
+          @Override
+          public void remove() {
+            throw new UnsupportedOperationException("Remove not supported");
+          }
+        };
       }
-    });
+    };
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/87f3ea35/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index 52173f6..814af6b 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -21,6 +21,7 @@ package org.apache.tez.dag.app.rm;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -29,6 +30,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.PriorityBlockingQueue;
@@ -147,7 +149,8 @@ public class TaskScheduler extends AbstractService
   Map<ContainerId, HeldContainer> heldContainers =
       new HashMap<ContainerId, HeldContainer>();
   
-  Set<NodeId> blacklistedNodes = Sets.newConcurrentHashSet();
+  Set<NodeId> blacklistedNodes = Collections
+      .newSetFromMap(new ConcurrentHashMap<NodeId, Boolean>());
   
   Resource totalResources = Resource.newInstance(0, 0);
   Resource allocatedResources = Resource.newInstance(0, 0);

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/87f3ea35/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
index 51d2de8..91a0b22 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/common/resources/MemoryDistributor.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
@@ -44,7 +45,6 @@ import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
 
 // Not calling this a MemoryManager explicitly. Not yet anyway.
 @Private
@@ -63,7 +63,8 @@ public class MemoryDistributor {
   private final boolean isEnabled;
   private final boolean reserveFractionConfigured;
   private float reserveFraction;
-  private final Set<TezTaskContext> dupSet = Sets.newConcurrentHashSet();
+  private final Set<TezTaskContext> dupSet = Collections
+      .newSetFromMap(new ConcurrentHashMap<TezTaskContext, Boolean>());
   private final List<RequestorInfo> requestList;
   
   // Maybe make the reserve fraction configurable. Or scale it based on JVM heap.


Mime
View raw message