falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srik...@apache.org
Subject [45/47] git commit: checkstyle related fixes for retention module.
Date Fri, 26 Apr 2013 15:51:01 GMT
checkstyle related fixes for retention module.


Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/4e39c7a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/4e39c7a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/4e39c7a9

Branch: refs/heads/master
Commit: 4e39c7a960a7ed0dd2d4a934e462ca92be074dd7
Parents: 6a39baf
Author: venkatesh <venkatesh@hortonworks.com>
Authored: Tue Apr 23 13:27:39 2013 -0700
Committer: venkatesh <venkatesh@hortonworks.com>
Committed: Tue Apr 23 13:27:39 2013 -0700

----------------------------------------------------------------------
 .../org/apache/falcon/retention/FeedEvictor.java   |   51 ++++----
 .../apache/falcon/retention/FeedEvictorTest.java   |  101 ++++++---------
 2 files changed, 65 insertions(+), 87 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4e39c7a9/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
----------------------------------------------------------------------
diff --git a/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java b/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
index 14c2c1c..155c847 100644
--- a/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
+++ b/retention/src/main/java/org/apache/falcon/retention/FeedEvictor.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -44,6 +44,7 @@ import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.*;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -53,27 +54,25 @@ import java.util.regex.Pattern;
  */
 public class FeedEvictor extends Configured implements Tool {
 
-    private static Logger LOG = Logger.getLogger(FeedEvictor.class);
+    private static final Logger LOG = Logger.getLogger(FeedEvictor.class);
 
-    private static final ExpressionEvaluator EVALUATOR = new
-            ExpressionEvaluatorImpl();
-    private static final ExpressionHelper resolver = ExpressionHelper.get();
+    private static final ExpressionEvaluator EVALUATOR = new ExpressionEvaluatorImpl();
+    private static final ExpressionHelper RESOLVER = ExpressionHelper.get();
 
-    static PrintStream stream = System.out;
+    public static final AtomicReference<PrintStream> OUT = new AtomicReference<PrintStream>(System.out);
+//    static PrintStream stream = System.out;
 
-    private static final String format = "yyyyMMddHHmm";
+    private static final String FORMAT = "yyyyMMddHHmm";
 
     public static void main(String[] args) throws Exception {
         Configuration conf = new Configuration();
         Path confPath = new Path("file:///" + System.getProperty("oozie.action.conf.xml"));
 
-        LOG.info(confPath + " found ? " +
-                confPath.getFileSystem(conf).exists(confPath));
+        LOG.info(confPath + " found ? " + confPath.getFileSystem(conf).exists(confPath));
         conf.addResource(confPath);
         int ret = ToolRunner.run(conf, new FeedEvictor(), args);
         if (ret != 0) {
-            throw new Exception("Unable to perform eviction action args: " +
-                    Arrays.toString(args));
+            throw new Exception("Unable to perform eviction action args: " + Arrays.toString(args));
         }
     }
 
@@ -98,12 +97,12 @@ public class FeedEvictor extends Configured implements Tool {
             evictor(path, retentionType, retentionLimit, timeZone, frequency, logFile);
         }
 
-        logInstancePaths(new Path(logFile), instancePaths.toString());
+        logInstancePaths(new Path(logFile));
         int len = buffer.length();
         if (len > 0) {
-            stream.println("instances=" + buffer.substring(0, len - 1));
+            OUT.get().println("instances=" + buffer.substring(0, len - 1));
         } else {
-            stream.println("instances=NULL");
+            OUT.get().println("instances=NULL");
         }
         return 0;
     }
@@ -124,7 +123,7 @@ public class FeedEvictor extends Configured implements Tool {
                 + retentionType + ", Limit: " + retentionLimit + ", timezone: "
                 + timeZone + ", frequency: " + frequency);
 
-        DateFormat dateFormat = new SimpleDateFormat(format);
+        DateFormat dateFormat = new SimpleDateFormat(FORMAT);
         dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
         for (Path path : toBeDeleted) {
             if (deleteInstance(path)) {
@@ -137,27 +136,26 @@ public class FeedEvictor extends Configured implements Tool {
 
     }
 
-    private void logInstancePaths(Path path, String instancePaths) throws IOException {
+    private void logInstancePaths(Path path) throws IOException {
         LOG.info("Writing deleted instances to path " + path);
         OutputStream out = fs.create(path);
-        out.write(instancePaths.getBytes());
+        out.write(instancePaths.toString().getBytes());
         out.close();
         if (LOG.isDebugEnabled()) {
-            debug(path, fs);
+            debug(path);
         }
     }
 
     private Pair<Date, Date> getDateRange(String period) throws ELException {
         Long duration = (Long) EVALUATOR.evaluate("${" + period + "}",
-                Long.class, resolver, resolver);
+                Long.class, RESOLVER, RESOLVER);
         Date end = new Date();
         Date start = new Date(end.getTime() - duration);
         return Pair.of(start, end);
     }
 
     private List<Path> discoverInstanceToDelete(String inPath, String timeZone,
-                                                String dateMask, Date start)
-            throws IOException {
+                                                String dateMask, Date start) throws IOException
{
 
         FileStatus[] files = findFilesForFeed(inPath);
         if (files == null || files.length == 0) {
@@ -187,8 +185,7 @@ public class FeedEvictor extends Configured implements Tool {
                 .replaceAll(VARS.MINUTE.regex(), "mm");
     }
 
-    private FileStatus[] findFilesForFeed(String feedBasePath)
-            throws IOException {
+    private FileStatus[] findFilesForFeed(String feedBasePath) throws IOException {
 
         Matcher matcher = FeedDataPath.PATTERN.matcher(feedBasePath);
         while (matcher.find()) {
@@ -234,7 +231,7 @@ public class FeedEvictor extends Configured implements Tool {
         }
 
         try {
-            DateFormat dateFormat = new SimpleDateFormat(format.
+            DateFormat dateFormat = new SimpleDateFormat(FORMAT.
                     substring(0, date.length()));
             dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
             return dateFormat.parse(date);
@@ -268,7 +265,7 @@ public class FeedEvictor extends Configured implements Tool {
         return fs.delete(path, true);
     }
 
-    private void debug(Path outPath, FileSystem fs) throws IOException {
+    private void debug(Path outPath) throws IOException {
         ByteArrayOutputStream writer = new ByteArrayOutputStream();
         InputStream instance = fs.open(outPath);
         IOUtils.copyBytes(instance, writer, 4096, true);
@@ -276,8 +273,7 @@ public class FeedEvictor extends Configured implements Tool {
         LOG.debug("Written " + writer);
     }
 
-    private CommandLine getCommand(String[] args)
-            throws org.apache.commons.cli.ParseException {
+    private CommandLine getCommand(String[] args) throws org.apache.commons.cli.ParseException
{
         Options options = new Options();
         Option opt;
         opt = new Option("feedBasePath", true,
@@ -304,5 +300,4 @@ public class FeedEvictor extends Configured implements Tool {
         options.addOption(opt);
         return new GnuParser().parse(options, args);
     }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/4e39c7a9/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index 66e99a6..c5a2013 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -1,4 +1,4 @@
-/*
+/**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -35,6 +35,9 @@ import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 
+/**
+ * Test for FeedEvictor.
+ */
 public class FeedEvictorTest {
 
     private EmbeddedCluster cluster;
@@ -44,7 +47,7 @@ public class FeedEvictorTest {
     @BeforeClass
     public void start() throws Exception {
         cluster = EmbeddedCluster.newCluster("test", false);
-        FeedEvictor.stream = stream;
+        FeedEvictor.OUT.set(stream);
     }
 
     @AfterClass
@@ -58,12 +61,14 @@ public class FeedEvictorTest {
             FeedEvictor.main(null);
             Assert.fail("Expected an exception to be thrown");
         } catch (Exception ignore) {
+            // ignore
         }
 
         try {
             FeedEvictor.main(new String[]{"1", "2"});
             Assert.fail("Expected an exception to be thrown");
         } catch (Exception ignore) {
+            // ignore
         }
     }
 
@@ -71,7 +76,8 @@ public class FeedEvictorTest {
     public void testEviction1() throws Exception {
         try {
             FeedEvictor.main(new String[]{"1", "2", "3", "4", "5", "6", "7"});
-        } catch (Exception e) {
+        } catch (Exception ignore) {
+            // ignore
         }
     }
 
@@ -89,18 +95,15 @@ public class FeedEvictorTest {
             String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
 
             FeedEvictor.main(new String[]{
-                    "-feedBasePath", cluster.getConf().get("fs.default.name")
-                    + dataPath,
-                    "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone",
"UTC", "-frequency",
-                    "daily",
-                    "-logFile", logFile});
+                "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
+                "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone",
"UTC", "-frequency",
+                "daily", "-logFile", logFile, });
 
             assertFailures(fs, pair);
             compare(map.get("feed1"), stream.getBuffer());
 
             Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
 
-
         } catch (Exception e) {
             Assert.fail("Unknown exception", e);
         }
@@ -170,15 +173,12 @@ public class FeedEvictorTest {
 
             Pair<List<String>, List<String>> pair;
             pair = createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS, "/data");
-            String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/" +
-                    "?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
+            String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}";
             String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
             FeedEvictor.main(new String[]{
-                    "-feedBasePath", cluster.getConf().get("fs.default.name")
-                    + dataPath,
-                    "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone",
"UTC", "-frequency",
-                    "hourly",
-                    "-logFile", logFile});
+                "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath,
+                "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone",
"UTC", "-frequency",
+                "hourly", "-logFile", logFile, });
             assertFailures(fs, pair);
 
             compare(map.get("feed2"), stream.getBuffer());
@@ -201,25 +201,19 @@ public class FeedEvictorTest {
 
             Pair<List<String>, List<String>> pair;
             pair = createTestData("/data");
-            FeedEvictor.main(new String[]{
-                    "-feedBasePath",
-                    cluster.getConf().get("fs.default.name")
-                            + "/data/YYYY/feed3/dd/MM/"
-                            + "?{MONTH}/more/?{HOUR}", "-retentionType",
-                    "instance", "-retentionLimit", "months(5)", "-timeZone",
-                    "UTC", "-frequency", "hourly", "-logFile",
-                    "/falcon/staging/feed/2012-01-01-04-00"});
+            FeedEvictor.main(new String[] {
+                "-feedBasePath",
+                cluster.getConf().get("fs.default.name") + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}",
+                "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
+                "UTC", "-frequency", "hourly", "-logFile", "/falcon/staging/feed/2012-01-01-04-00",
});
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             stream.clear();
-            String dataPath = "/data/YYYY/feed4/dd/MM/" +
-                    "02/more/hello";
+            String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
             String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
-            FeedEvictor.main(new String[]{"-feedBasePath",
-                                          cluster.getConf().get("fs.default.name") + dataPath,
-                                          "-retentionType", "instance", "-retentionLimit",
-                                          "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
-                                          "-logFile", logFile});
+            FeedEvictor.main(new String[] {"-feedBasePath",
+                cluster.getConf().get("fs.default.name") + dataPath, "-retentionType", "instance",
+                "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
"-logFile", logFile, });
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
@@ -243,26 +237,23 @@ public class FeedEvictorTest {
             statsPair = createTestData("/stats");
             metaPair = createTestData("/meta");
             tmpPair = createTestData("/tmp");
-            FeedEvictor.main(new String[]{
-                    "-feedBasePath",
-                    getFeedBasePath(cluster, "/data") + "#"
-                            + getFeedBasePath(cluster, "/stats") + "#"
-                            + getFeedBasePath(cluster, "/meta") + "#"
-                            + getFeedBasePath(cluster, "/tmp"),
-                    "-retentionType", "instance", "-retentionLimit",
-                    "months(5)", "-timeZone", "UTC", "-frequency", "hourly",
-                    "-logFile", "/falcon/staging/feed/2012-01-01-04-00"});
+            FeedEvictor.main(new String[] {"-feedBasePath",
+                getFeedBasePath("/data") + "#"
+                    + getFeedBasePath("/stats") + "#"
+                    + getFeedBasePath("/meta") + "#"
+                    + getFeedBasePath("/tmp"),
+                "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone",
+                "UTC", "-frequency", "hourly", "-logFile", "/falcon/staging/feed/2012-01-01-04-00",
});
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             stream.clear();
-            String dataPath = "/data/YYYY/feed4/dd/MM/" +
-                    "02/more/hello";
+            String dataPath = "/data/YYYY/feed4/dd/MM/02/more/hello";
             String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-02-00.csv";
             FeedEvictor.main(new String[]{"-feedBasePath",
                                           cluster.getConf().get("fs.default.name") + dataPath,
                                           "-retentionType", "instance", "-retentionLimit",
                                           "hours(5)", "-timeZone", "UTC", "-frequency", "hourly",
-                                          "-logFile", logFile});
+                                          "-logFile", logFile, });
             Assert.assertEquals("instances=NULL", stream.getBuffer());
 
             Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath));
@@ -295,12 +286,8 @@ public class FeedEvictorTest {
                     + "/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}";
             String logFile = "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
 
-            FeedEvictor.main(new String[]{
-                    "-feedBasePath",
-                    dataPath,
-                    "-retentionType", "instance", "-retentionLimit", "days(10)", "-timeZone",
"UTC", "-frequency",
-                    "daily",
-                    "-logFile", logFile});
+            FeedEvictor.main(new String[] {"-feedBasePath", dataPath, "-retentionType", "instance",
+                "-retentionLimit", "days(10)", "-timeZone", "UTC", "-frequency", "daily",
"-logFile", logFile, });
 
             assertFailures(fs, pair);
 
@@ -333,12 +320,9 @@ public class FeedEvictorTest {
         return Pair.of(inRange, outOfRange);
     }
 
-    private Pair<List<String>, List<String>> createTestData(String feed,
-                                                            String mask,
-                                                            int period,
-                                                            TimeUnit timeUnit, String locationType)
-            throws Exception {
-
+    private Pair<List<String>, List<String>> createTestData(String feed,
String mask,
+                                                            int period, TimeUnit timeUnit,
+                                                            String locationType) throws Exception
{
         Configuration conf = cluster.getConf();
         FileSystem fs = FileSystem.get(conf);
 
@@ -370,12 +354,11 @@ public class FeedEvictorTest {
                 SimpleDateFormat(timeUnit == TimeUnit.HOURS ? "yyyyMMddHH" : "yyyyMMdd");
         displayFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
 
-        StringBuffer buffer = new StringBuffer();
+        StringBuilder buffer = new StringBuilder();
         for (long date = now;
              date > now - timeUnit.toMillis(range + 6);
              date -= timeUnit.toMillis(1)) {
-            String path = locationType + "/YYYY/" + feed + "/mmHH/dd/MM/" +
-                    format.format(date);
+            String path = locationType + "/YYYY/" + feed + "/mmHH/dd/MM/" + format.format(date);
             touch(fs, path);
             if (date <= now && date > now - timeUnit.toMillis(range)) {
                 outOfRange.add(path);
@@ -393,7 +376,7 @@ public class FeedEvictorTest {
         fs.create(new Path(path)).close();
     }
 
-    private String getFeedBasePath(EmbeddedCluster cluster, String locationType) {
+    private String getFeedBasePath(String locationType) {
         return cluster.getConf().get("fs.default.name")
                 + "/data/YYYY/feed3/dd/MM/"
                 + "?{MONTH}/more/?{HOUR}";


Mime
View raw message