falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From suh...@apache.org
Subject falcon git commit: FALCON-1146 feed retention policy deleted everything all the way up to the root. Contributed by Peeyush Bishnoi
Date Mon, 13 Apr 2015 18:12:24 GMT
Repository: falcon
Updated Branches:
  refs/heads/0.6.1 7ec516ed4 -> 8cbe93e9c


FALCON-1146 feed retention policy deleted everything all the way up to the root. Contributed
by Peeyush Bishnoi


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/8cbe93e9
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/8cbe93e9
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/8cbe93e9

Branch: refs/heads/0.6.1
Commit: 8cbe93e9cf6bbd3ed46ba5bec4d2bc45c696f377
Parents: 7ec516e
Author: Suhas Vasu <suhas.v@inmobi.com>
Authored: Mon Apr 13 23:42:02 2015 +0530
Committer: Suhas Vasu <suhas.v@inmobi.com>
Committed: Mon Apr 13 23:42:02 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  3 +
 .../apache/falcon/entity/FileSystemStorage.java |  2 +-
 .../falcon/retention/FeedEvictorTest.java       | 61 +++++++++++++++-----
 3 files changed, 52 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/8cbe93e9/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 77a0fef..02b8be2 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -137,6 +137,9 @@ Branch: 0.6.1 (Proposed Release Version: 0.6.1)
    (Suhas vasu)
 
   BUG FIXES
+   FALCON-1146 feed retention policy deleted everything all the way
+   up to the root (Peeyush Bishnoi via Suhas Vasu)
+
    FALCON-1129 In a secure cluster, feed replication fails because of
    Authentication issues (Venkat Ranganathan via Suhas Vasu)
 

http://git-wip-us.apache.org/repos/asf/falcon/blob/8cbe93e9/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index 8948ad4..9ff33e7 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -328,7 +328,7 @@ public class FileSystemStorage extends Configured implements Storage {
 
         DateFormat dateFormat = new SimpleDateFormat(FeedHelper.FORMAT);
         dateFormat.setTimeZone(timeZone);
-        Path feedBasePath = FeedHelper.getFeedBasePath(feedPath);
+        Path feedBasePath = fs.makeQualified(FeedHelper.getFeedBasePath(feedPath));
         for (Path path : toBeDeleted) {
             deleteInstance(fs, path, feedBasePath);
             Date date = FeedHelper.getDate(feedPath, new Path(path.toUri().getPath()), timeZone);

http://git-wip-us.apache.org/repos/asf/falcon/blob/8cbe93e9/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index a2feccf..e3daf8c 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -24,6 +24,7 @@ import org.apache.falcon.entity.Storage;
 import org.apache.falcon.entity.v0.feed.LocationType;
 import org.apache.falcon.hadoop.HadoopClientFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IOUtils;
@@ -32,19 +33,10 @@ import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PrintStream;
+import java.io.*;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TimeZone;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -370,7 +362,7 @@ public class FeedEvictorTest {
         }
     }
 
-    @Test(enabled = false)
+    @Test
     public void testEvictionWithEmptyDirs() throws Exception {
         try {
             Configuration conf = cluster.getConf();
@@ -398,7 +390,7 @@ public class FeedEvictorTest {
 
             compare(map.get("feed1"), stream.getBuffer());
 
-            String expectedInstancePaths = getExpectedInstancePaths(dataPath.replaceAll(storageUrl,
""));
+            String expectedInstancePaths = getExpectedInstancePaths(dataPath);
             Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths);
 
             String deletedPath = expectedInstancePaths.split(",")[0].split("=")[1];
@@ -421,6 +413,49 @@ public class FeedEvictorTest {
         }
     }
 
+    @Test
+    public void testFeedBasePathExists() throws Exception {
+        try {
+            Configuration conf = cluster.getConf();
+            FileSystem fs = FileSystem.get(conf);
+            fs.delete(new Path("/"), true);
+            stream.clear();
+
+            Pair<List<String>, List<String>> pair = generateInstances(fs,
"feed3",
+                    "yyyy/MM/dd/hh/mm", 1, TimeUnit.MINUTES, "/data", false);
+            final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
+            String dataPath = LocationType.DATA.name() + "="
+                    + storageUrl + "/data/YYYY/feed3/mmHH/dd/MM/?{YEAR}/?{MONTH}/?{DAY}/?{HOUR}/?{MINUTE}";
+            String logFile = hdfsUrl + "/falcon/staging/feed/instancePaths-2012-01-01-01-00.csv";
+            long beforeDelCount = fs.getContentSummary(new Path(("/data/YYYY/feed3/mmHH/dd/MM/"))).getDirectoryCount();
+
+            FeedEvictor.main(new String[]{
+                "-feedBasePath", dataPath,
+                "-retentionType", "instance",
+                "-retentionLimit", "minutes(-1)",
+                "-timeZone", "UTC",
+                "-frequency", "minutes",
+                "-logFile", logFile,
+                "-falconFeedStorageType", Storage.TYPE.FILESYSTEM.name(),
+            });
+
+            String expectedInstancePaths = getExpectedInstancePaths(dataPath);
+            Assert.assertEquals(readLogFile(new Path(logFile)), expectedInstancePaths);
+
+            //Feed Base path must exist
+            Assert.assertTrue(fs.exists(new Path("/data/YYYY/feed3/mmHH/dd/MM/")));
+            FileStatus[] files = fs.listStatus(new Path(("/data/YYYY/feed3/mmHH/dd/MM/")));
+            //Number of directories/files inside feed base path should be 0
+            Assert.assertEquals(files.length, 0);
+            long afterDelCount = fs.getContentSummary(new Path(("/data/YYYY/feed3/mmHH/dd/MM/"))).getDirectoryCount();
+            //Number of directories deleted
+            Assert.assertEquals((beforeDelCount - afterDelCount), 11);
+
+        } catch (Exception e) {
+            Assert.fail("Unknown exception", e);
+        }
+    }
+
 
     private Pair<List<String>, List<String>> createTestData(String locationType)
throws Exception {
         Configuration conf = cluster.getConf();


Mime
View raw message