tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jeag...@apache.org
Subject [07/50] [abbrv] tez git commit: TEZ-3330. Propagate additional config parameters when running MR jobs via Tez. (sseth)
Date Tue, 06 Dec 2016 17:06:38 GMT
TEZ-3330. Propagate additional config parameters when running MR jobs via Tez. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/e9fb4186
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/e9fb4186
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/e9fb4186

Branch: refs/heads/TEZ-3334
Commit: e9fb418647dec7253956a1a62fbbcd22175144e4
Parents: 539b8e7
Author: Siddharth Seth <sseth@apache.org>
Authored: Mon Oct 10 10:56:16 2016 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Mon Oct 10 10:56:16 2016 -0700

----------------------------------------------------------------------
 CHANGES.txt                                         |  2 ++
 .../org/apache/tez/mapreduce/client/YARNRunner.java |  3 ++-
 .../tez/runtime/library/common/ConfigUtils.java     | 10 ++++++++++
 .../tez/runtime/library/conf/BaseConfigBuilder.java |  2 ++
 .../library/conf/OrderedGroupedKVInputConfig.java   | 16 ++++++++++++++++
 .../conf/OrderedPartitionedKVEdgeConfig.java        |  7 +++++++
 .../conf/OrderedPartitionedKVOutputConfig.java      | 16 ++++++++++++++++
 .../runtime/library/conf/UnorderedKVEdgeConfig.java |  7 +++++++
 .../library/conf/UnorderedKVInputConfig.java        | 15 +++++++++++++++
 .../library/conf/UnorderedKVOutputConfig.java       | 16 ++++++++++++++++
 .../conf/UnorderedPartitionedKVEdgeConfig.java      |  8 ++++++++
 .../conf/UnorderedPartitionedKVOutputConfig.java    | 16 ++++++++++++++++
 .../conf/TestOrderedGroupedMergedKVInputConfig.java |  7 ++++++-
 .../conf/TestOrderedPartitionedKVEdgeConfig.java    |  9 ++++++++-
 .../conf/TestOrderedPartitionedKVOutputConfig.java  |  7 ++++++-
 .../library/conf/TestUnorderedKVEdgeConfig.java     |  7 ++++++-
 .../library/conf/TestUnorderedKVInputConfig.java    |  7 ++++++-
 .../library/conf/TestUnorderedKVOutputConfig.java   |  7 ++++++-
 .../conf/TestUnorderedPartitionedKVEdgeConfig.java  |  7 ++++++-
 .../TestUnorderedPartitionedKVOutputConfig.java     |  7 ++++++-
 20 files changed, 167 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index addd2c0..10fbd92 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3330. Propagate additional config parameters when running MR jobs via Tez.
   TEZ-3335. DAG client thinks app is still running when app status is null
   TEZ-3437. Improve synchronization and the progress report behavior for Inputs from TEZ-3317.
   TEZ-3460. Fix precommit release audit warning.
@@ -119,6 +120,7 @@ INCOMPATIBLE CHANGES
 
 ALL CHANGES:
 
+  TEZ-3330. Propagate additional config parameters when running MR jobs via Tez.
   TEZ-3335. DAG client thinks app is still running when app status is null
   TEZ-3460. Fix precommit release audit warning.
   TEZ-3368. NPE in DelayedContainerManager

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index 1bea781..820e2e4 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -515,8 +515,9 @@ public class YARNRunner implements ClientProtocol {
                     TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS),
                 stageConfs[i - 1].get(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS),
                 MRPartitioner.class.getName(), partitionerConf)
+                .setFromConfigurationUnfiltered(stageConfs[i-1])
                 .configureInput().useLegacyInput().done()
-                .setFromConfiguration(stageConfs[i - 1]).build();
+                .build();
         Edge edge = Edge.create(vertices[i - 1], vertices[i], edgeConf.createDefaultEdgeProperty());
         dag.addEdge(edge);
       }

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java
index c63f737..6aa797f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/ConfigUtils.java
@@ -227,6 +227,16 @@ public class ConfigUtils {
     }
   }
 
+  public static void mergeConfs(Configuration destConf, Configuration srcConf) {
+    Preconditions.checkState(destConf != null, "Destination conf cannot be null");
+    Preconditions.checkState(srcConf != null, "Source conf cannot be null");
+    for (Map.Entry<String, String> entry : srcConf) {
+      // Explicit get to have parameter replacement work.
+      String val = srcConf.get(entry.getKey());
+      destConf.set(entry.getKey(), val);
+    }
+  }
+
   private static Map<String, String> extractConfigurationMapInternal(
       Iterable<Map.Entry<String, String>> iterable, List<Set<String>>
validKeySets, List<String> allowedPrefixes) {
     Set<String> validKeys = new HashSet<String>();

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java
index d991e89..162addf 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/BaseConfigBuilder.java
@@ -59,4 +59,6 @@ interface BaseConfigBuilder<T> {
    */
   @InterfaceAudience.LimitedPrivate({"hive, pig"})
   public T setFromConfiguration(Configuration conf);
+
+  public T setFromConfigurationUnfiltered(Configuration conf);
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
index 95d8e75..11a8d6f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedGroupedKVInputConfig.java
@@ -212,6 +212,13 @@ public class OrderedGroupedKVInputConfig {
       return this;
     }
 
+    @Override
+    public SpecificBuilder setFromConfigurationUnfiltered(
+        Configuration conf) {
+      builder.setFromConfigurationUnfiltered(conf);
+      return this;
+    }
+
     public E done() {
       return edgeBuilder;
     }
@@ -448,6 +455,15 @@ public class OrderedGroupedKVInputConfig {
       return this;
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    public Builder setFromConfigurationUnfiltered(Configuration conf) {
+      // Maybe ensure this is the first call ? Otherwise this can end up overriding other
parameters
+      Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+      ConfigUtils.mergeConfs(this.conf, conf);
+      return this;
+    }
+
     public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
                                   @Nullable Map<String, String> codecConf) {
       this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
index e6cc2c2..3504203 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
@@ -284,6 +284,13 @@ public class OrderedPartitionedKVEdgeConfig
       return this;
     }
 
+    @Override
+    public Builder setFromConfigurationUnfiltered(Configuration conf) {
+      outputBuilder.setFromConfigurationUnfiltered(conf);
+      inputBuilder.setFromConfigurationUnfiltered(conf);
+      return this;
+    }
+
     /**
      * Configure the specific output
      * @return a builder to configure the output

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
index 0687e79..0f37c66 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVOutputConfig.java
@@ -177,6 +177,13 @@ public class OrderedPartitionedKVOutputConfig {
       return this;
     }
 
+    @Override
+    public SpecificBuilder setFromConfigurationUnfiltered(
+        Configuration conf) {
+      builder.setFromConfigurationUnfiltered(conf);
+      return this;
+    }
+
     public E done() {
       return edgeBuilder;
     }
@@ -374,6 +381,15 @@ public class OrderedPartitionedKVOutputConfig {
       return this;
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    public Builder setFromConfigurationUnfiltered(Configuration conf) {
+      // Maybe ensure this is the first call ? Otherwise this can end up overriding other
parameters
+      Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+      ConfigUtils.mergeConfs(this.conf, conf);
+      return this;
+    }
+
     /**
      * Set the key comparator class
      *

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
index 28a7503..25a4823 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
@@ -219,6 +219,13 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig
{
       return this;
     }
 
+    @Override
+    public Builder setFromConfigurationUnfiltered(Configuration conf) {
+      outputBuilder.setFromConfigurationUnfiltered(conf);
+      inputBuilder.setFromConfigurationUnfiltered(conf);
+      return this;
+    }
+
     /**
      * Set serialization class responsible for providing serializer/deserializer for key/value
and
      * the corresponding comparator class to be used as key comparator.

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
index f52c672..af7dbf6 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVInputConfig.java
@@ -139,6 +139,12 @@ public class UnorderedKVInputConfig {
       return this;
     }
 
+    @Override
+    public SpecificBuilder setFromConfigurationUnfiltered(Configuration conf) {
+      builder.setFromConfigurationUnfiltered(conf);
+      return this;
+    }
+
     public E done() {
       return edgeBuilder;
     }
@@ -296,6 +302,15 @@ public class UnorderedKVInputConfig {
       return this;
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    public Builder setFromConfigurationUnfiltered(Configuration conf) {
+      // Maybe ensure this is the first call ? Otherwise this can end up overriding other
parameters
+      Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+      ConfigUtils.mergeConfs(this.conf, conf);
+      return this;
+    }
+
     public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
                                   @Nullable Map<String, String> codecConf) {
       this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
index e3eeedc..a9a15a0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVOutputConfig.java
@@ -86,6 +86,13 @@ public class UnorderedKVOutputConfig {
       return this;
     }
 
+    @Override
+    public SpecificBuilder setFromConfigurationUnfiltered(
+        Configuration conf) {
+      builder.setFromConfigurationUnfiltered(conf);
+      return this;
+    }
+
     public E done() {
       return edgeBuilder;
     }
@@ -222,6 +229,15 @@ public class UnorderedKVOutputConfig {
       return this;
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    public Builder setFromConfigurationUnfiltered(Configuration conf) {
+      // Maybe ensure this is the first call ? Otherwise this can end up overriding other
parameters
+      Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+      ConfigUtils.mergeConfs(this.conf, conf);
+      return this;
+    }
+
     /**
      * Set serialization class responsible for providing serializer/deserializer for keys.
      *

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
index 8ac7d65..52da491 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
@@ -224,6 +224,14 @@ public class UnorderedPartitionedKVEdgeConfig
       return this;
     }
 
+    @Override
+    public Builder setFromConfigurationUnfiltered(
+        Configuration conf) {
+      outputBuilder.setFromConfigurationUnfiltered(conf);
+      inputBuilder.setFromConfigurationUnfiltered(conf);
+      return this;
+    }
+
     /**
      * Set serialization class responsible for providing serializer/deserializer for keys.
      *

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
index 1cf0924..3555e1c 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVOutputConfig.java
@@ -100,6 +100,13 @@ public class UnorderedPartitionedKVOutputConfig {
       return this;
     }
 
+    @Override
+    public SpecificBuilder setFromConfigurationUnfiltered(
+        Configuration conf) {
+      builder.setFromConfigurationUnfiltered(conf);
+      return this;
+    }
+
     public E done() {
       return edgeBuilder;
     }
@@ -266,6 +273,15 @@ public class UnorderedPartitionedKVOutputConfig {
       return this;
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    public Builder setFromConfigurationUnfiltered(Configuration conf) {
+      // Maybe ensure this is the first call ? Otherwise this can end up overriding other
parameters
+      Preconditions.checkArgument(conf != null, "Configuration cannot be null");
+      ConfigUtils.mergeConfs(this.conf, conf);
+      return this;
+    }
+
     public Builder setCompression(boolean enabled, @Nullable String compressionCodec,
                                   @Nullable Map<String, String> codecConf) {
       this.conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, enabled);

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java
index 49b5490..70dbe14 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedGroupedMergedKVInputConfig.java
@@ -75,6 +75,8 @@ public class TestOrderedGroupedMergedKVInputConfig {
         .TEZ_RUNTIME_SHUFFLE_MIN_REQUIRED_PROGRESS_FRACTION, 0.6f);
     fromConf.setBoolean(TezRuntimeConfiguration
         .TEZ_RUNTIME_SHUFFLE_FAILED_CHECK_SINCE_LAST_COMPLETION, false);
+    Configuration fromConfUnfiltered = new Configuration(false);
+    fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
     Map<String, String> additionalConf = new HashMap<String, String>();
     additionalConf.put("test.key.2", "key2");
     additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3");
@@ -92,7 +94,8 @@ public class TestOrderedGroupedMergedKVInputConfig {
             .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
                 String.valueOf(false))
             .setAdditionalConfiguration(additionalConf)
-            .setFromConfiguration(fromConf);
+            .setFromConfiguration(fromConf)
+            .setFromConfigurationUnfiltered(fromConfUnfiltered);
 
     OrderedGroupedKVInputConfig configuration = builder.build();
 
@@ -139,6 +142,8 @@ public class TestOrderedGroupedMergedKVInputConfig {
     assertNull(conf.get("test.conf.key.1"));
     assertNull(conf.get("test.key.1"));
     assertNull(conf.get("test.key.2"));
+
+    assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
   }
 
   @Test(timeout = 5000)

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
index 9d6ca50..9567770 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVEdgeConfig.java
@@ -128,6 +128,8 @@ public class TestOrderedPartitionedKVEdgeConfig {
         true);
     fromConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB,
         1000);
+    Configuration fromConfUnfiltered = new Configuration(false);
+    fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
     Map<String, String> additionalConfs = new HashMap<String, String>();
     additionalConfs.put("test.key.2", "key2");
     additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111");
@@ -145,7 +147,8 @@ public class TestOrderedPartitionedKVEdgeConfig {
         .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT,
"0.33f")
         .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES,
"3333")
         .setAdditionalConfiguration(additionalConfs)
-        .setFromConfiguration(fromConf);
+        .setFromConfiguration(fromConf)
+        .setFromConfigurationUnfiltered(fromConfUnfiltered);
 
     OrderedPartitionedKVEdgeConfig configuration = builder.build();
 
@@ -175,6 +178,9 @@ public class TestOrderedPartitionedKVEdgeConfig {
     assertEquals("io", outputConf.get("io.shouldExist"));
     assertEquals("file", outputConf.get("file.shouldExist"));
     assertEquals("fs", outputConf.get("fs.shouldExist"));
+
+    assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1"));
+
     ReportPartitionStats partitionStats =
         ReportPartitionStats.fromString(outputConf.get(
         TezRuntimeConfiguration.TEZ_RUNTIME_REPORT_PARTITION_STATS,
@@ -198,6 +204,7 @@ public class TestOrderedPartitionedKVEdgeConfig {
     assertEquals("file", inputConf.get("file.shouldExist"));
     assertEquals("fs", inputConf.get("fs.shouldExist"));
 
+    assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1"));
   }
 
   @Test (timeout=2000)

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
index 891e807..2dca183 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestOrderedPartitionedKVOutputConfig.java
@@ -73,6 +73,8 @@ public class TestOrderedPartitionedKVOutputConfig {
     additionalConf.put("test.key.2", "key2");
     additionalConf.put("io.shouldExist", "io");
     additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "TestInternalSorter");
+    Configuration fromConfUnfiltered = new Configuration(false);
+    fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
     OrderedPartitionedKVOutputConfig.Builder builder =
         OrderedPartitionedKVOutputConfig.newBuilder("KEY", "VALUE", "PARTITIONER", null)
             .setKeyComparatorClass("KEY_COMPARATOR", null)
@@ -83,7 +85,8 @@ public class TestOrderedPartitionedKVOutputConfig {
             .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
                 String.valueOf(false))
             .setAdditionalConfiguration(additionalConf)
-            .setFromConfiguration(fromConf);
+            .setFromConfiguration(fromConf)
+            .setFromConfigurationUnfiltered(fromConfUnfiltered);
 
     OrderedPartitionedKVOutputConfig configuration = builder.build();
 
@@ -116,6 +119,8 @@ public class TestOrderedPartitionedKVOutputConfig {
     assertNull(conf.get("test.conf.key.1"));
     assertNull(conf.get("test.key.1"));
     assertNull(conf.get("test.key.2"));
+
+    assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
   }
 
   @Test(timeout = 5000)

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
index 0ef1cc5..3c97020 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVEdgeConfig.java
@@ -121,6 +121,8 @@ public class TestUnorderedKVEdgeConfig {
     fromConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, false);
     fromConf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCH_BUFFER_PERCENT, 0.11f);
     fromConf.set("io.shouldExist", "io");
+    Configuration fromConfUnfiltered = new Configuration(false);
+    fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
     Map<String, String> additionalConfs = new HashMap<String, String>();
     additionalConfs.put("test.key.2", "key2");
     additionalConfs.put(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES, "1111");
@@ -135,7 +137,8 @@ public class TestUnorderedKVEdgeConfig {
         .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE,
"3333")
         .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT,
"0.33f")
         .setAdditionalConfiguration(additionalConfs)
-        .setFromConfiguration(fromConf);
+        .setFromConfiguration(fromConf)
+        .setFromConfigurationUnfiltered(fromConfUnfiltered);
 
     UnorderedKVEdgeConfig configuration = builder.build();
 
@@ -159,6 +162,7 @@ public class TestUnorderedKVEdgeConfig {
     assertEquals("file", outputConf.get("file.shouldExist"));
     assertEquals("fs", outputConf.get("fs.shouldExist"));
 
+    assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1"));
 
     assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
true));
     assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES,
0));
@@ -173,6 +177,7 @@ public class TestUnorderedKVEdgeConfig {
     assertEquals("file", inputConf.get("file.shouldExist"));
     assertEquals("fs", inputConf.get("fs.shouldExist"));
 
+    assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1"));
   }
 
   private void checkHistoryText(String historyText) {

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
index 8c7dea4..bb75442 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVInputConfig.java
@@ -62,6 +62,8 @@ public class TestUnorderedKVInputConfig {
     additionalConf.put("test.key.2", "key2");
     additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, "3");
     additionalConf.put("file.shouldExist", "file");
+    Configuration fromConfUnfiltered = new Configuration(false);
+    fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
     UnorderedKVInputConfig.Builder builder =
         UnorderedKVInputConfig.newBuilder("KEY", "VALUE")
             .setCompression(true, "CustomCodec", null)
@@ -73,7 +75,8 @@ public class TestUnorderedKVInputConfig {
             .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
                 String.valueOf(false))
             .setAdditionalConfiguration(additionalConf)
-            .setFromConfiguration(fromConf);
+            .setFromConfiguration(fromConf)
+            .setFromConfigurationUnfiltered(fromConfUnfiltered);
 
     UnorderedKVInputConfig configuration = builder.build();
 
@@ -105,6 +108,8 @@ public class TestUnorderedKVInputConfig {
     assertNull(conf.get("test.conf.key.1"));
     assertNull(conf.get("test.key.1"));
     assertNull(conf.get("test.key.2"));
+
+    assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
   }
 
   @Test(timeout = 5000)

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java
index 10ed199..f99b3ae 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedKVOutputConfig.java
@@ -63,6 +63,8 @@ public class TestUnorderedKVOutputConfig {
     additionalConf.put("test.key.2", "key2");
     additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES,
"2222");
     additionalConf.put("file.shouldExist", "file");
+    Configuration fromConfUnfiltered = new Configuration(false);
+    fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
     UnorderedKVOutputConfig.Builder builder =
         UnorderedKVOutputConfig.newBuilder("KEY", "VALUE")
             .setCompression(true, "CustomCodec", null)
@@ -71,7 +73,8 @@ public class TestUnorderedKVOutputConfig {
             .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
                 String.valueOf(false))
             .setAdditionalConfiguration(additionalConf)
-            .setFromConfiguration(fromConf);
+            .setFromConfiguration(fromConf)
+            .setFromConfigurationUnfiltered(fromConfUnfiltered);
 
     UnorderedKVOutputConfig configuration = builder.build();
 
@@ -100,6 +103,8 @@ public class TestUnorderedKVOutputConfig {
     assertNull(conf.get("test.conf.key.1"));
     assertNull(conf.get("test.key.1"));
     assertNull(conf.get("test.key.2"));
+
+    assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
   }
 
   @Test(timeout = 5000)

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
index fff8efd..684f773 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVEdgeConfig.java
@@ -136,6 +136,8 @@ public class TestUnorderedPartitionedKVEdgeConfig {
     additionalConfs
         .put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES,
"2222");
     additionalConfs.put("file.shouldExist", "file");
+    Configuration fromConfUnfiltered = new Configuration(false);
+    fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
 
     UnorderedPartitionedKVEdgeConfig.Builder builder = UnorderedPartitionedKVEdgeConfig
         .newBuilder("KEY", "VALUE", "PARTITIONER")
@@ -144,7 +146,8 @@ public class TestUnorderedPartitionedKVEdgeConfig {
         .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IO_FILE_BUFFER_SIZE,
"3333")
         .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_MERGE_PERCENT,
"0.33f")
         .setAdditionalConfiguration(additionalConfs)
-        .setFromConfiguration(fromConf);
+        .setFromConfiguration(fromConf)
+        .setFromConfigurationUnfiltered(fromConfUnfiltered);
 
     UnorderedPartitionedKVEdgeConfig configuration = builder.build();
 
@@ -172,6 +175,7 @@ public class TestUnorderedPartitionedKVEdgeConfig {
     assertEquals("file", outputConf.get("file.shouldExist"));
     assertEquals("fs", outputConf.get("fs.shouldExist"));
 
+    assertEquals("unfiltered1", outputConf.get("test.conf.unfiltered.1"));
 
     assertEquals(false, inputConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
true));
     assertEquals(1111, inputConf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD_BYTES,
0));
@@ -188,6 +192,7 @@ public class TestUnorderedPartitionedKVEdgeConfig {
     assertEquals("file", inputConf.get("file.shouldExist"));
     assertEquals("fs", inputConf.get("fs.shouldExist"));
 
+    assertEquals("unfiltered1", inputConf.get("test.conf.unfiltered.1"));
   }
 
   private void checkHistoryText(String historyText) {

http://git-wip-us.apache.org/repos/asf/tez/blob/e9fb4186/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java
index f509acb..a9e955e 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/conf/TestUnorderedPartitionedKVOutputConfig.java
@@ -73,6 +73,8 @@ public class TestUnorderedPartitionedKVOutputConfig {
     additionalConf.put("test.key.2", "key2");
     additionalConf.put(TezRuntimeConfiguration.TEZ_RUNTIME_UNORDERED_OUTPUT_MAX_PER_BUFFER_SIZE_BYTES,
"2222");
     additionalConf.put("file.shouldExist", "file");
+    Configuration fromConfUnfiltered = new Configuration(false);
+    fromConfUnfiltered.set("test.conf.unfiltered.1", "unfiltered1");
     UnorderedPartitionedKVOutputConfig.Builder builder =
         UnorderedPartitionedKVOutputConfig.newBuilder("KEY", "VALUE", "PARTITIONER",
             null)
@@ -87,7 +89,8 @@ public class TestUnorderedPartitionedKVOutputConfig {
             .setAdditionalConfiguration(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD,
                 String.valueOf(false))
             .setAdditionalConfiguration(additionalConf)
-            .setFromConfiguration(fromConf);
+            .setFromConfiguration(fromConf)
+            .setFromConfigurationUnfiltered(fromConfUnfiltered);
 
     UnorderedPartitionedKVOutputConfig configuration = builder.build();
 
@@ -125,6 +128,8 @@ public class TestUnorderedPartitionedKVOutputConfig {
     assertNull(conf.get("test.conf.key.1"));
     assertNull(conf.get("test.key.1"));
     assertNull(conf.get("test.key.2"));
+
+    assertEquals("unfiltered1", conf.get("test.conf.unfiltered.1"));
   }
 
   @Test(timeout = 5000)


Mime
View raw message