hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vvasu...@apache.org
Subject [41/50] [abbrv] hadoop git commit: YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)
Date Fri, 27 Jan 2017 18:40:46 GMT
YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ed968a1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ed968a1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ed968a1

Branch: refs/heads/YARN-3926
Commit: 6ed968a1fd89bab240b60af56b4d72a9b2e725b5
Parents: 285fce5
Author: Arun Suresh <asuresh@apache.org>
Authored: Thu Mar 17 23:50:22 2016 -0700
Committer: Varun Vasudev <vvasudev@apache.org>
Committed: Fri Jan 27 23:23:36 2017 +0530

----------------------------------------------------------------------
 .../yarn/api/records/ResourceInformation.java   |  2 +-
 .../hadoop/yarn/util/UnitsConversionUtil.java   | 45 ++++++++++++++------
 .../yarn/util/TestUnitsConversionUtil.java      | 17 +++++++-
 .../api/records/impl/pb/ResourcePBImpl.java     |  2 +-
 .../yarn/util/resource/TestResourceUtils.java   |  2 +-
 5 files changed, 52 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed968a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 80e3192..a17e81b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -36,7 +36,7 @@ public class ResourceInformation implements Comparable<ResourceInformation>
{
   private static final String VCORES_URI = "vcores";
 
   public static final ResourceInformation MEMORY_MB =
-      ResourceInformation.newInstance(MEMORY_URI, "M");
+      ResourceInformation.newInstance(MEMORY_URI, "Mi");
   public static final ResourceInformation VCORES =
       ResourceInformation.newInstance(VCORES_URI);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed968a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
index 7785263..47bb3df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
@@ -46,7 +46,8 @@ public class UnitsConversionUtil {
   }
 
   private static final String[] UNITS =
-      {"p", "n", "u", "m", "", "k", "M", "G", "T", "P"};
+      { "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", "Ti",
+          "Pi" };
   private static final List<String> SORTED_UNITS = Arrays.asList(UNITS);
   public static final Set<String> KNOWN_UNITS = createKnownUnitsSet();
   private static final Converter PICO =
@@ -65,6 +66,15 @@ public class UnitsConversionUtil {
   private static final Converter PETA =
       new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L);
 
+  private static final Converter KILO_BINARY = new Converter(1024L, 1L);
+  private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 1L);
+  private static final Converter GIGA_BINARY =
+      new Converter(1024L * 1024L * 1024L, 1L);
+  private static final Converter TERA_BINARY =
+      new Converter(1024L * 1024L * 1024L * 1024L, 1L);
+  private static final Converter PETA_BINARY =
+      new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L);
+
   private static Set<String> createKnownUnitsSet() {
     Set<String> ret = new HashSet<>();
     ret.addAll(Arrays.asList(UNITS));
@@ -93,6 +103,16 @@ public class UnitsConversionUtil {
       return TERA;
     case "P":
       return PETA;
+    case "Ki":
+      return KILO_BINARY;
+    case "Mi":
+      return MEGA_BINARY;
+    case "Gi":
+      return GIGA_BINARY;
+    case "Ti":
+      return TERA_BINARY;
+    case "Pi":
+      return PETA_BINARY;
     default:
       throw new IllegalArgumentException(
           "Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS);
@@ -112,28 +132,29 @@ public class UnitsConversionUtil {
     if (toUnit == null || fromUnit == null || fromValue == null) {
       throw new IllegalArgumentException("One or more arguments are null");
     }
-    Long tmp;
     String overflowMsg =
         "Converting " + fromValue + " from '" + fromUnit + "' to '" + toUnit
             + "' will result in an overflow of Long";
+    if (fromUnit.equals(toUnit)) {
+      return fromValue;
+    }
     Converter fc = getConverter(fromUnit);
     Converter tc = getConverter(toUnit);
     Long numerator = fc.numerator * tc.denominator;
     Long denominator = fc.denominator * tc.numerator;
+    Long numeratorMultiplierLimit = Long.MAX_VALUE / numerator;
     if (numerator < denominator) {
-      if (!toUnit.equals(fromUnit)) {
-        tmp = Long.MAX_VALUE / numerator;
-        if (tmp < fromValue) {
-          throw new IllegalArgumentException(overflowMsg);
-        }
+      if (numeratorMultiplierLimit < fromValue) {
+        throw new IllegalArgumentException(overflowMsg);
       }
       return (fromValue * numerator) / denominator;
     }
-    tmp = numerator / denominator;
-    if (!toUnit.equals(fromUnit)) {
-      if ((Long.MAX_VALUE / tmp) < fromValue) {
-        throw new IllegalArgumentException(overflowMsg);
-      }
+    if (numeratorMultiplierLimit > fromValue) {
+      return (numerator * fromValue) / denominator;
+    }
+    Long tmp = numerator / denominator;
+    if ((Long.MAX_VALUE / tmp) < fromValue) {
+      throw new IllegalArgumentException(overflowMsg);
     }
     return fromValue * tmp;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed968a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
index 421768f..0f999e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/util/TestUnitsConversionUtil.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.util;
 
-import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -45,6 +44,7 @@ public class TestUnitsConversionUtil {
     fromUnit = "";
     Assert.assertEquals("kilo test failed", Long.valueOf(test / 1000l),
         UnitsConversionUtil.convert(fromUnit, "k", test));
+
     Assert
         .assertEquals("mega test failed", Long.valueOf(test / (1000l * 1000l)),
             UnitsConversionUtil.convert(fromUnit, "M", test));
@@ -63,6 +63,21 @@ public class TestUnitsConversionUtil {
 
     Assert.assertEquals("mega to giga test failed", Long.valueOf(value),
         UnitsConversionUtil.convert("M", "G", Long.valueOf(value * 1000l)));
+
+    Assert.assertEquals("Mi to Gi test failed", Long.valueOf(value),
+        UnitsConversionUtil.convert("Mi", "Gi", Long.valueOf(value * 1024l)));
+
+    Assert.assertEquals("Mi to Ki test failed", Long.valueOf(value * 1024),
+        UnitsConversionUtil.convert("Mi", "Ki", Long.valueOf(value)));
+
+    Assert.assertEquals("Ki to base units test failed", Long.valueOf(5 * 1024),
+        UnitsConversionUtil.convert("Ki", "", Long.valueOf(5)));
+
+    Assert.assertEquals("Mi to k test failed", Long.valueOf(1073741),
+        UnitsConversionUtil.convert("Mi", "k", Long.valueOf(1024)));
+
+    Assert.assertEquals("M to Mi test failed", Long.valueOf(953),
+        UnitsConversionUtil.convert("M", "Mi", Long.valueOf(1000)));
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed968a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index c0ae52f..6ff091c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -81,7 +81,7 @@ public class ResourcePBImpl extends Resource {
     initResourcesMap();
     ResourceInformation ri =
         this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
-    return UnitsConversionUtil.convert(ri.getUnits(), "M", ri.getValue());
+    return UnitsConversionUtil.convert(ri.getUnits(), "Mi", ri.getValue());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed968a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 606e89c..4bb68c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -195,7 +195,7 @@ public class TestResourceUtils {
       // we must always have memory and vcores with their fixed units
       Assert.assertTrue(ret.containsKey("memory-mb"));
       ResourceInformation memInfo = ret.get("memory-mb");
-      Assert.assertEquals("M", memInfo.getUnits());
+      Assert.assertEquals("Mi", memInfo.getUnits());
       Assert.assertEquals(ResourceTypes.COUNTABLE, memInfo.getResourceType());
       Assert.assertTrue(ret.containsKey("vcores"));
       ResourceInformation vcoresInfo = ret.get("vcores");


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message