tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject tez git commit: TEZ-2203. Intern strings in tez counters (bikas) (cherry picked from commit 6e15b2f6dac8471d266b8f313038f3767fedab04)
Date Mon, 21 Sep 2015 21:34:15 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 0f6c1086f -> 3e5991a5c


TEZ-2203. Intern strings in tez counters (bikas)
(cherry picked from commit 6e15b2f6dac8471d266b8f313038f3767fedab04)

Dropping TestMockDAGAppMaster.java since the test is ignored and mainly
for manual memory testing.

Conflicts:
	CHANGES.txt
	tez-dag/src/test/java/org/apache/tez/dag/app/TestMockDAGAppMaster.java
(cherry picked from commit edc2720980712dfcafcc944482f1e116b1d50030)

Conflicts:
	CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/3e5991a5
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/3e5991a5
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/3e5991a5

Branch: refs/heads/branch-0.5
Commit: 3e5991a5cba5e81764ee5778e408fd4226cc0806
Parents: 0f6c108
Author: Bikas Saha <bikas@apache.org>
Authored: Fri Mar 20 12:27:02 2015 -0700
Committer: Bikas Saha <bikas@apache.org>
Committed: Mon Sep 21 14:33:51 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                             |  1 +
 .../tez/common/counters/AbstractCounterGroup.java       |  7 ++++---
 .../tez/common/counters/FileSystemCounterGroup.java     |  9 ++++++---
 .../tez/common/counters/FrameworkCounterGroup.java      |  3 +--
 .../org/apache/tez/common/counters/GenericCounter.java  | 12 ++++++------
 5 files changed, 18 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f9e4734..0646dfc 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
   TEZ-2552. CRC errors can cause job to run for very long time in large jobs.
 
 ALL CHANGES:
+  TEZ-2203. Intern strings in tez counters
   TEZ-2834. Make Tez preemption resilient to incorrect free resource reported
   by YARN
   TEZ-2812. Preemption sometimes does not respect heartbeats between preemptions

http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
index 7e791d7..a4b153f 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/AbstractCounterGroup.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentSkipListMap;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringInterner;
 
 import com.google.common.collect.Iterators;
 
@@ -49,8 +50,8 @@ public abstract class AbstractCounterGroup<T extends TezCounter>
 
   public AbstractCounterGroup(String name, String displayName,
                               Limits limits) {
-    this.name = name;
-    this.displayName = displayName;
+    this.name = StringInterner.weakIntern(name);
+    this.displayName = StringInterner.weakIntern(displayName);
     this.limits = limits;
   }
 
@@ -160,7 +161,7 @@ public abstract class AbstractCounterGroup<T extends TezCounter>
 
   @Override
   public synchronized void readFields(DataInput in) throws IOException {
-    displayName = Text.readString(in);
+    displayName = StringInterner.weakIntern(Text.readString(in));
     counters.clear();
     int size = WritableUtils.readVInt(in);
     for (int i = 0; i < size; i++) {

http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
b/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
index 88d47ca..5024154 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/FileSystemCounterGroup.java
@@ -29,13 +29,16 @@ import java.util.Locale;
 import java.util.Map;
 
 import com.google.common.base.Joiner;
+
 import static com.google.common.base.Preconditions.*;
+
 import com.google.common.collect.AbstractIterator;
 import com.google.common.collect.Iterators;
 import com.google.common.collect.Maps;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringInterner;
 
 /**
  * An abstract class to provide common implementation of the filesystem
@@ -54,7 +57,7 @@ public abstract class FileSystemCounterGroup<C extends TezCounter>
   // Just a few local casts probably worth not having to carry it around.
   private final Map<String, Object[]> map =
     new ConcurrentSkipListMap<String, Object[]>();
-  private String displayName = "File System Counters";
+  private String displayName = StringInterner.weakIntern("File System Counters");
 
   private static final Joiner NAME_JOINER = Joiner.on('_');
 
@@ -65,7 +68,7 @@ public abstract class FileSystemCounterGroup<C extends TezCounter>
     private long value;
 
     public FSCounter(String scheme, FileSystemCounter ref) {
-      this.scheme = scheme;
+      this.scheme = scheme; // this is interned in the checkScheme() method via a map
       key = ref;
     }
 
@@ -122,7 +125,7 @@ public abstract class FileSystemCounterGroup<C extends TezCounter>
 
   @Override
   public void setDisplayName(String displayName) {
-    this.displayName = displayName;
+    this.displayName = StringInterner.weakIntern(displayName);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
b/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
index 66b6e33..3a4aa97 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/FrameworkCounterGroup.java
@@ -26,7 +26,6 @@ import java.io.IOException;
 import java.util.Arrays;
 import java.util.Iterator;
 
-import com.google.common.base.Joiner;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.WritableUtils;
 
@@ -60,7 +59,7 @@ public abstract class FrameworkCounterGroup<T extends Enum<T>,
 
     public FrameworkCounter(T ref, String groupName) {
       key = ref;
-      this.groupName = groupName;
+      this.groupName = groupName; // this is interned in the fmap/i2s of CounterGroupFactory
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/tez/blob/3e5991a5/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java b/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java
index 5477606..4bb4c76 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/GenericCounter.java
@@ -25,6 +25,7 @@ import java.io.IOException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringInterner;
 
 /**
  * A generic counter implementation
@@ -41,13 +42,12 @@ public class GenericCounter extends AbstractCounter {
   }
 
   public GenericCounter(String name, String displayName) {
-    this.name = name;
-    this.displayName = displayName;
+    this(name, displayName, 0);
   }
 
   public GenericCounter(String name, String displayName, long value) {
-    this.name = name;
-    this.displayName = displayName;
+    this.name = StringInterner.weakIntern(name);
+    this.displayName = StringInterner.weakIntern(displayName);
     this.value = value;
   }
 
@@ -58,8 +58,8 @@ public class GenericCounter extends AbstractCounter {
 
   @Override
   public synchronized void readFields(DataInput in) throws IOException {
-    name = Text.readString(in);
-    displayName = in.readBoolean() ? Text.readString(in) : name;
+    name = StringInterner.weakIntern(Text.readString(in));
+    displayName = in.readBoolean() ? StringInterner.weakIntern(Text.readString(in)) : name;
     value = WritableUtils.readVLong(in);
   }
 


Mime
View raw message