falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srik...@apache.org
Subject [43/47] git commit: checkstyle related fixes for replication module.
Date Fri, 26 Apr 2013 15:50:59 GMT
checkstyle related fixes for replication module.


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/dbd980ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/dbd980ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/dbd980ea

Branch: refs/heads/master
Commit: dbd980ea4f97fce2182689d12667cdb04915388e
Parents: fcfd9b3
Author: venkatesh <venkatesh@hortonworks.com>
Authored: Tue Apr 23 13:26:55 2013 -0700
Committer: venkatesh <venkatesh@hortonworks.com>
Committed: Tue Apr 23 13:26:55 2013 -0700

----------------------------------------------------------------------
 .../falcon/replication/CustomReplicator.java       |    7 ++-
 .../apache/falcon/replication/FeedReplicator.java  |    8 ++-
 .../falcon/replication/FilteredCopyListing.java    |   15 +++--
 .../falcon/repliation/FeedReplicatorTest.java      |   49 --------------
 .../falcon/replication/FeedReplicatorTest.java     |   52 +++++++++++++++
 .../replication/FilteredCopyListingTest.java       |   11 ++-
 6 files changed, 80 insertions(+), 62 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/dbd980ea/replication/src/main/java/org/apache/falcon/replication/CustomReplicator.java
----------------------------------------------------------------------
diff --git a/replication/src/main/java/org/apache/falcon/replication/CustomReplicator.java
b/replication/src/main/java/org/apache/falcon/replication/CustomReplicator.java
index 7dfd406..ba01c80 100644
--- a/replication/src/main/java/org/apache/falcon/replication/CustomReplicator.java
+++ b/replication/src/main/java/org/apache/falcon/replication/CustomReplicator.java
@@ -27,9 +27,14 @@ import org.apache.log4j.Logger;
 
 import java.io.IOException;
 
+/**
+ * A custom implementation of DistCp that overrides the behavior of CopyListing
+ * interface to copy FileOutputCommitter.SUCCEEDED_FILE_NAME last so downstream apps
+ * depending on data availability will work correctly.
+ */
 public class CustomReplicator extends DistCp {
 
-    private static Logger LOG = Logger.getLogger(CustomReplicator.class);
+    private static final Logger LOG = Logger.getLogger(CustomReplicator.class);
 
     /**
      * Public Constructor. Creates DistCp object with specified input-parameters.

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/dbd980ea/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java
----------------------------------------------------------------------
diff --git a/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java b/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java
index fc0b5ac..efb5691 100644
--- a/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java
+++ b/replication/src/main/java/org/apache/falcon/replication/FeedReplicator.java
@@ -35,9 +35,12 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Pattern;
 
+/**
+ * A tool for feed replication that uses DistCp tool to replicate.
+ */
 public class FeedReplicator extends Configured implements Tool {
 
-    private static Logger LOG = Logger.getLogger(FeedReplicator.class);
+    private static final Logger LOG = Logger.getLogger(FeedReplicator.class);
 
     public static void main(String[] args) throws Exception {
         ToolRunner.run(new Configuration(), new FeedReplicator(), args);
@@ -82,8 +85,7 @@ public class FeedReplicator extends Configured implements Tool {
             }
         } else {
             LOG.info("No files present in path: "
-                    + new Path(targetPath.toString() + "/" + fixedPath)
-                    .toString());
+                    + new Path(targetPath.toString() + "/" + fixedPath).toString());
         }
         LOG.info("Completed DistCp");
         return 0;

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/dbd980ea/replication/src/main/java/org/apache/falcon/replication/FilteredCopyListing.java
----------------------------------------------------------------------
diff --git a/replication/src/main/java/org/apache/falcon/replication/FilteredCopyListing.java
b/replication/src/main/java/org/apache/falcon/replication/FilteredCopyListing.java
index c1698e3..9c97a44 100644
--- a/replication/src/main/java/org/apache/falcon/replication/FilteredCopyListing.java
+++ b/replication/src/main/java/org/apache/falcon/replication/FilteredCopyListing.java
@@ -29,6 +29,11 @@ import org.apache.log4j.Logger;
 import java.io.IOException;
 import java.util.regex.Pattern;
 
+/**
+ * An implementation of CopyListing that overrides the default behavior by suppressing file,
+ * FileOutputCommitter.SUCCEEDED_FILE_NAME and copies that in the last so downstream apps
+ * depending on data availability will work correctly.
+ */
 public class FilteredCopyListing extends SimpleCopyListing {
     private static final Logger LOG = Logger.getLogger(FilteredCopyListing.class);
 
@@ -54,8 +59,8 @@ public class FilteredCopyListing extends SimpleCopyListing {
             LOG.info("Inclusion pattern = " + configuration.get("falcon.include.path"));
             LOG.info("Regex pattern = " + regex);
         } catch (IOException e) {
-            throw new IllegalArgumentException("Unable to build regex for " +
-                    configuration.get("falcon.include.path", ""));
+            throw new IllegalArgumentException("Unable to build regex for "
+                    + configuration.get("falcon.include.path", ""));
         }
     }
 
@@ -68,8 +73,8 @@ public class FilteredCopyListing extends SimpleCopyListing {
     }
 
     private static boolean isJavaRegexSpecialChar(char pChar) {
-        return pChar == '.' || pChar == '$' || pChar == '(' || pChar == ')' ||
-                pChar == '|' || pChar == '+';
+        return pChar == '.' || pChar == '$' || pChar == '(' || pChar == ')'
+                || pChar == '|' || pChar == '+';
     }
 
     public static Pattern getRegEx(String filePattern) throws IOException {
@@ -122,7 +127,7 @@ public class FilteredCopyListing extends SimpleCopyListing {
                 pCh = ')';
             } else if (pCh == '[' && setOpen == 0) {
                 setOpen++;
-            } else if (pCh == '^' && setOpen > 0) {
+            // } else if (pCh == '^' && setOpen > 0) {
             } else if (pCh == '-' && setOpen > 0) {
                 // Character set range
                 setRange = true;

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/dbd980ea/replication/src/test/java/org/apache/falcon/repliation/FeedReplicatorTest.java
----------------------------------------------------------------------
diff --git a/replication/src/test/java/org/apache/falcon/repliation/FeedReplicatorTest.java
b/replication/src/test/java/org/apache/falcon/repliation/FeedReplicatorTest.java
deleted file mode 100644
index 1935e51..0000000
--- a/replication/src/test/java/org/apache/falcon/repliation/FeedReplicatorTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.falcon.repliation;
-
-import org.apache.falcon.replication.FeedReplicator;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.tools.DistCpOptions;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class FeedReplicatorTest {
-
-    @Test
-    public void testArguments() throws Exception {
-        /*
-         * <arg>-update</arg>
-         * <arg>-blocking</arg><arg>true</arg> <arg>-maxMaps</arg><arg>20</arg>
-         * <arg>-sourcePaths</arg><arg>${distcpSourcePaths}</arg>
-         * <arg>-targetPath</arg><arg>${distcpTargetPaths}</arg>
-         */
-        FeedReplicator replicator = new FeedReplicator();
-        DistCpOptions options = replicator.getDistCpOptions(new String[]{"true", "-maxMaps",
"5", "-sourcePaths",
-                                                                         "hdfs://localhost:8020/tmp/",
"-targetPath",
-                                                                         "hdfs://localhost1:8020/tmp/"});
-
-        List<Path> srcPaths = new ArrayList<Path>();
-        srcPaths.add(new Path("hdfs://localhost:8020/tmp/"));
-        Assert.assertEquals(options.getSourcePaths(), srcPaths);
-        Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost1:8020/tmp/"));
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/dbd980ea/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java
----------------------------------------------------------------------
diff --git a/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java
b/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java
new file mode 100644
index 0000000..67795b5
--- /dev/null
+++ b/replication/src/test/java/org/apache/falcon/replication/FeedReplicatorTest.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.falcon.replication;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.tools.DistCpOptions;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Test class for FeedReplicator.
+ */
+public class FeedReplicatorTest {
+
+    @Test
+    public void testArguments() throws Exception {
+        /*
+         * <arg>-update</arg>
+         * <arg>-blocking</arg><arg>true</arg> <arg>-maxMaps</arg><arg>20</arg>
+         * <arg>-sourcePaths</arg><arg>${distcpSourcePaths}</arg>
+         * <arg>-targetPath</arg><arg>${distcpTargetPaths}</arg>
+         */
+        FeedReplicator replicator = new FeedReplicator();
+        DistCpOptions options = replicator.getDistCpOptions(new String[] {
+            "true", "-maxMaps", "5", "-sourcePaths",
+            "hdfs://localhost:8020/tmp/", "-targetPath",
+            "hdfs://localhost1:8020/tmp/", });
+
+        List<Path> srcPaths = new ArrayList<Path>();
+        srcPaths.add(new Path("hdfs://localhost:8020/tmp/"));
+        Assert.assertEquals(options.getSourcePaths(), srcPaths);
+        Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost1:8020/tmp/"));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/dbd980ea/replication/src/test/java/org/apache/falcon/replication/FilteredCopyListingTest.java
----------------------------------------------------------------------
diff --git a/replication/src/test/java/org/apache/falcon/replication/FilteredCopyListingTest.java
b/replication/src/test/java/org/apache/falcon/replication/FilteredCopyListingTest.java
index 5054bf8..e308866 100644
--- a/replication/src/test/java/org/apache/falcon/replication/FilteredCopyListingTest.java
+++ b/replication/src/test/java/org/apache/falcon/replication/FilteredCopyListingTest.java
@@ -38,11 +38,14 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 
+/**
+ * Test class for org.apache.falcon.replication.FilteredCopyListing.
+ */
 public class FilteredCopyListingTest {
 
     private static final Credentials CREDENTIALS = new Credentials();
 
-    public static Map<String, String> expectedValues = new HashMap<String, String>();
+    public static final Map<String, String> EXPECTED_VALUES = new HashMap<String,
String>();
 
     @BeforeClass
     public static void setup() throws Exception {
@@ -101,7 +104,7 @@ public class FilteredCopyListingTest {
     private static void recordInExpectedValues(String path) throws Exception {
         FileSystem fileSystem = FileSystem.getLocal(new Configuration());
         Path sourcePath = new Path(fileSystem.getUri().toString() + path);
-        expectedValues.put(sourcePath.toString(), DistCpUtils.getRelativePath(
+        EXPECTED_VALUES.put(sourcePath.toString(), DistCpUtils.getRelativePath(
                 new Path("/tmp/source"), sourcePath));
     }
 
@@ -215,9 +218,9 @@ public class FilteredCopyListingTest {
             actualValues.put(value.getPath().toString(), key.toString());
         }
 
-        Assert.assertEquals(expected == -1 ? expectedValues.size() : expected, actualValues.size());
+        Assert.assertEquals(expected == -1 ? EXPECTED_VALUES.size() : expected, actualValues.size());
         for (Map.Entry<String, String> entry : actualValues.entrySet()) {
-            Assert.assertEquals(entry.getValue(), expectedValues.get(entry.getKey()));
+            Assert.assertEquals(entry.getValue(), EXPECTED_VALUES.get(entry.getKey()));
         }
     }
 }


Mime
View raw message