falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkat...@apache.org
Subject [1/5] incubator-falcon git commit: FALCON-815 Rename mapBandwidthKB to mapBandwidth in DistCp. Contributed by Sowmya Ramesh
Date Sat, 08 Nov 2014 05:32:35 GMT
Repository: incubator-falcon
Updated Branches:
  refs/heads/master 81904565a -> 394055a54


FALCON-815 Rename mapBandwidthKB to mapBandwidth in DistCp. Contributed by Sowmya Ramesh


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/112f9009
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/112f9009
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/112f9009

Branch: refs/heads/master
Commit: 112f90094df224e47760300bf768d264cbea11ff
Parents: d2a2508
Author: Venkatesh Seetharam <venkatesh@apache.org>
Authored: Fri Nov 7 20:21:13 2014 -0800
Committer: Venkatesh Seetharam <venkatesh@apache.org>
Committed: Fri Nov 7 21:33:25 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                                    | 3 +++
 .../src/main/resources/hdfs-replication-workflow.xml           | 4 ++--
 .../src/main/resources/hdfs-replication.properties             | 4 ++--
 common/src/main/resources/runtime.properties                   | 2 +-
 docs/src/site/twiki/EntitySpecification.twiki                  | 4 ++--
 .../falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java   | 4 ++--
 oozie/src/main/resources/action/feed/replication-action.xml    | 4 ++--
 .../apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java | 4 ++--
 oozie/src/test/resources/feed/fs-replication-feed.xml          | 2 +-
 .../java/org/apache/falcon/replication/FeedReplicator.java     | 6 +++---
 .../java/org/apache/falcon/replication/FeedReplicatorTest.java | 6 +++---
 11 files changed, 23 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index aaea457..7abc820 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -129,6 +129,9 @@ Trunk (Unreleased)
   OPTIMIZATIONS
 
   BUG FIXES
+   FALCON-815 Rename mapBandwidthKB to mapBandwidth in DistCp
+   (Sowmya Ramesh via Venkatesh Seetharam)
+
    FALCON-595 Improvements to DefaultAuthorizationProvider code
    (Raghav Kumar Gautam via Venkatesh Seetharam)
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
index 841b9df..4ef8fc7 100644
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication-workflow.xml
@@ -49,8 +49,8 @@
             <arg>-Dmapred.job.priority=${jobPriority}</arg>
             <arg>-maxMaps</arg>
             <arg>${maxMaps}</arg>
-            <arg>-mapBandwidthKB</arg>
-            <arg>${mapBandwidthKB}</arg>
+            <arg>-mapBandwidth</arg>
+            <arg>${mapBandwidth}</arg>
             <arg>-sourcePaths</arg>
             <arg>${nameNode}${drSourceDir}</arg>
             <arg>-targetPath</arg>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
----------------------------------------------------------------------
diff --git a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
index 29d6cab..19b8459 100644
--- a/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
+++ b/addons/recipes/hdfs-replication/src/main/resources/hdfs-replication.properties
@@ -61,5 +61,5 @@ falcon.recipe.process.property5.name=drTargetCluster
 falcon.recipe.process.property5.value=backupCluster
 falcon.recipe.process.property6.name=maxMaps
 falcon.recipe.process.property6.value=5
-falcon.recipe.process.property7.name=mapBandwidthKB
-falcon.recipe.process.property7.value=102400
+falcon.recipe.process.property7.name=mapBandwidth
+falcon.recipe.process.property7.value=100

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/common/src/main/resources/runtime.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/runtime.properties b/common/src/main/resources/runtime.properties
index 185602c..1d35667 100644
--- a/common/src/main/resources/runtime.properties
+++ b/common/src/main/resources/runtime.properties
@@ -27,4 +27,4 @@
 *.falcon.parentworkflow.retry.interval.secs=1
 
 *.falcon.replication.workflow.maxmaps=5
-*.falcon.replication.workflow.mapbandwidthKB=102400
+*.falcon.replication.workflow.mapbandwidth=100

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/docs/src/site/twiki/EntitySpecification.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/EntitySpecification.twiki b/docs/src/site/twiki/EntitySpecification.twiki
index 1e603ec..a81a626 100644
--- a/docs/src/site/twiki/EntitySpecification.twiki
+++ b/docs/src/site/twiki/EntitySpecification.twiki
@@ -278,13 +278,13 @@ permission indicates the permission.
         <property name="timeout" value="hours(1)"/>
         <property name="parallel" value="3"/>
         <property name="maxMaps" value="8"/>
-        <property name="mapBandwidthKB" value="1024"/>
+        <property name="mapBandwidth" value="1"/>
     </properties>
 </verbatim>
 A key-value pair, which are propagated to the workflow engine. "queueName" and "jobPriority"
are special properties
 available to user to specify the Hadoop job queue and priority, the same value is used by
Falcons launcher job.
 "timeout" and "parallel" are other special properties which decides replication instance's
timeout value while
-waiting for the feed instance and parallel decides the concurrent replication instances that
can run at any given time. "maxMaps" represents the maximum number of maps used during replication.
"mapBandwidthKB" represents the bandwidth in KB/s used by each mapper during replication.
+waiting for the feed instance and parallel decides the concurrent replication instances that
can run at any given time. "maxMaps" represents the maximum number of maps used during replication.
"mapBandwidth" represents the bandwidth in MB/s used by each mapper during replication.
  
 ---++ Process Specification
 A process defines configuration for a workflow. A workflow is a directed acyclic graph(DAG)
which defines the job for the workflow engine. A process definition defines  the configurations
required to run the workflow job. For example, process defines the frequency at which the
workflow should run, the clusters on which the workflow should run, the inputs and outputs
for the workflow, how the workflow failures should be handled, how the late inputs should
be handled and so on.  

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
index c5366dc..8f7f01a 100644
--- a/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
+++ b/oozie/src/main/java/org/apache/falcon/oozie/feed/FeedReplicationCoordinatorBuilder.java
@@ -73,7 +73,7 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F
     private static final String PARALLEL = "parallel";
     private static final String TIMEOUT = "timeout";
     private static final String MR_MAX_MAPS = "maxMaps";
-    private static final String MR_MAP_BANDWIDTH = "mapBandwidthKB";
+    private static final String MR_MAP_BANDWIDTH = "mapBandwidth";
 
     public FeedReplicationCoordinatorBuilder(Feed entity) {
         super(entity, LifeCycle.REPLICATION);
@@ -205,7 +205,7 @@ public class FeedReplicationCoordinatorBuilder extends OozieCoordinatorBuilder<F
     }
 
     private String getDefaultMapBandwidth() {
-        return RuntimeProperties.get().getProperty("falcon.replication.workflow.mapbandwidthKB",
"102400");
+        return RuntimeProperties.get().getProperty("falcon.replication.workflow.mapbandwidth",
"100");
     }
 
     private String getPathsWithPartitions(Cluster srcCluster, Cluster trgCluster) throws
FalconException {

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/oozie/src/main/resources/action/feed/replication-action.xml
----------------------------------------------------------------------
diff --git a/oozie/src/main/resources/action/feed/replication-action.xml b/oozie/src/main/resources/action/feed/replication-action.xml
index 5d154b4..da40b74 100644
--- a/oozie/src/main/resources/action/feed/replication-action.xml
+++ b/oozie/src/main/resources/action/feed/replication-action.xml
@@ -44,8 +44,8 @@
         <arg>-Dmapred.job.priority=${jobPriority}</arg>
         <arg>-maxMaps</arg>
         <arg>${maxMaps}</arg>
-        <arg>-mapBandwidthKB</arg>
-        <arg>${mapBandwidthKB}</arg>
+        <arg>-mapBandwidth</arg>
+        <arg>${mapBandwidth}</arg>
         <arg>-sourcePaths</arg>
         <arg>${distcpSourcePaths}</arg>
         <arg>-targetPath</arg>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
----------------------------------------------------------------------
diff --git a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
index 74cc509..42c231f 100644
--- a/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
+++ b/oozie/src/test/java/org/apache/falcon/oozie/feed/OozieFeedWorkflowBuilderTest.java
@@ -229,7 +229,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
         Assert.assertEquals(props.get("queueName"), "default");
         Assert.assertEquals(props.get("jobPriority"), "NORMAL");
         Assert.assertEquals(props.get("maxMaps"), "5");
-        Assert.assertEquals(props.get("mapBandwidthKB"), "102400");
+        Assert.assertEquals(props.get("mapBandwidth"), "100");
 
         assertLibExtensions(coord, "replication");
         WORKFLOWAPP wf = getWorkflowapp(trgMiniDFS.getFileSystem(), coord);
@@ -336,7 +336,7 @@ public class OozieFeedWorkflowBuilderTest extends AbstractTestBase {
         Assert.assertEquals(props.get("distcpTargetPaths"), "${coord:dataOut('output')}");
         Assert.assertEquals(props.get("falconFeedStorageType"), Storage.TYPE.FILESYSTEM.name());
         Assert.assertEquals(props.get("maxMaps"), "33");
-        Assert.assertEquals(props.get("mapBandwidthKB"), "2048");
+        Assert.assertEquals(props.get("mapBandwidth"), "2");
 
         verifyEntityProperties(aFeed, aCluster, srcCluster,
                 WorkflowExecutionContext.EntityOperations.REPLICATE, props);

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/oozie/src/test/resources/feed/fs-replication-feed.xml
----------------------------------------------------------------------
diff --git a/oozie/src/test/resources/feed/fs-replication-feed.xml b/oozie/src/test/resources/feed/fs-replication-feed.xml
index bada507..0e9065c 100644
--- a/oozie/src/test/resources/feed/fs-replication-feed.xml
+++ b/oozie/src/test/resources/feed/fs-replication-feed.xml
@@ -63,6 +63,6 @@
     <schema provider="protobuf" location="/databus/streams_local/click_rr/schema/"/>
     <properties>
         <property name="maxMaps" value="33" />
-        <property name="mapBandwidthKB" value="2048" />
+        <property name="mapBandwidth" value="2" />
     </properties>
 </feed>

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java
----------------------------------------------------------------------
diff --git a/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java b/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java
index 2bf2a98..90ac753 100644
--- a/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java
+++ b/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java
@@ -89,8 +89,8 @@ public class FeedReplicator extends Configured implements Tool {
         opt.setRequired(true);
         options.addOption(opt);
 
-        opt = new Option("mapBandwidthKB", true,
-                "bandwidth per map (in KB) to use for this copy");
+        opt = new Option("mapBandwidth", true,
+                "bandwidth per map (in MB) to use for this copy");
         opt.setRequired(true);
         options.addOption(opt);
 
@@ -119,7 +119,7 @@ public class FeedReplicator extends Configured implements Tool {
         distcpOptions.setSyncFolder(true);
         distcpOptions.setBlocking(true);
         distcpOptions.setMaxMaps(Integer.valueOf(cmd.getOptionValue("maxMaps")));
-        distcpOptions.setMapBandwidth(Integer.valueOf(cmd.getOptionValue("mapBandwidthKB")));
+        distcpOptions.setMapBandwidth(Integer.valueOf(cmd.getOptionValue("mapBandwidth")));
         return distcpOptions;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/112f9009/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java
----------------------------------------------------------------------
diff --git a/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java
b/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java
index d64eb61..539d00d 100644
--- a/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java
+++ b/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java
@@ -38,14 +38,14 @@ public class FeedReplicatorTest {
          * <arg>-update</arg>
          * <arg>-blocking</arg><arg>true</arg>
          * <arg>-maxMaps</arg><arg>3</arg>
-         * <arg>-mapBandwidthKB</arg><arg>4</arg>
+         * <arg>-mapBandwidth</arg><arg>4</arg>
          * <arg>-sourcePaths</arg><arg>${distcpSourcePaths}</arg>
          * <arg>-targetPath</arg><arg>${distcpTargetPaths}</arg>
          */
         final String[] args = {
             "true",
             "-maxMaps", "3",
-            "-mapBandwidthKB", "4096",
+            "-mapBandwidth", "4",
             "-sourcePaths", "hdfs://localhost:8020/tmp/",
             "-targetPath", "hdfs://localhost1:8020/tmp/",
             "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
@@ -58,7 +58,7 @@ public class FeedReplicatorTest {
         List<Path> srcPaths = new ArrayList<Path>();
         srcPaths.add(new Path("hdfs://localhost:8020/tmp/"));
         Assert.assertEquals(options.getMaxMaps(), 3);
-        Assert.assertEquals(options.getMapBandwidth(), 4096);
+        Assert.assertEquals(options.getMapBandwidth(), 4);
         Assert.assertEquals(options.getSourcePaths(), srcPaths);
         Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost1:8020/tmp/"));
     }


Mime
View raw message