falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-2023 Feed eviction fails when feed locations stats and meta …
Date Mon, 13 Jun 2016 22:19:13 GMT
Repository: falcon
Updated Branches:
  refs/heads/0.10 af7407f4c -> df6b53c83


FALCON-2023 Feed eviction fails when feed locations stats and meta …

…does not have time pattern

Author: Venkatesan Ramachandran <vramachandran@hortonworks.com>

Reviewers: " sandeepSamudrala <sandysmdl@gmail.com>, Ying Zheng <yzheng@hortonworks.com>,
Balu Vellanki <balu@apache.org>"

Closes #182 from vramachan/FALCON-2023


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/df6b53c8
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/df6b53c8
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/df6b53c8

Branch: refs/heads/0.10
Commit: df6b53c83520c9cd015dfbd0d752f25d18cd9c3f
Parents: af7407f
Author: Venkatesan Ramachandran <vramachandran@hortonworks.com>
Authored: Mon Jun 13 15:19:05 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Mon Jun 13 15:19:05 2016 -0700

----------------------------------------------------------------------
 .../apache/falcon/entity/FileSystemStorage.java | 11 +++-
 .../falcon/retention/FeedEvictorTest.java       | 53 ++++++++++++++++++++
 2 files changed, 62 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/df6b53c8/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
index ece8b5d..eb15585 100644
--- a/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/FileSystemStorage.java
@@ -358,13 +358,20 @@ public class FileSystemStorage extends Configured implements Storage
{
 
     private FileStatus[] findFilesForFeed(FileSystem fs, String feedBasePath) throws IOException
{
         Matcher matcher = FeedDataPath.PATTERN.matcher(feedBasePath);
+        boolean regexMatchFound = false;
         while (matcher.find()) {
+            regexMatchFound = true;
             String var = feedBasePath.substring(matcher.start(), matcher.end());
             feedBasePath = feedBasePath.replaceAll(Pattern.quote(var), "*");
             matcher = FeedDataPath.PATTERN.matcher(feedBasePath);
         }
-        LOG.info("Searching for {}", feedBasePath);
-        return fs.globStatus(new Path(feedBasePath));
+        if (regexMatchFound) {
+            LOG.info("Searching for {}", feedBasePath);
+            return fs.globStatus(new Path(feedBasePath));
+        } else {
+            LOG.info("Ignoring static path {}", feedBasePath);
+            return null;
+        }
     }
 
     private boolean isDateInRange(Date date, Date start) {

http://git-wip-us.apache.org/repos/asf/falcon/blob/df6b53c8/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
----------------------------------------------------------------------
diff --git a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
index 72447da..98936ae 100644
--- a/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
+++ b/retention/src/test/java/org/apache/falcon/retention/FeedEvictorTest.java
@@ -462,6 +462,43 @@ public class FeedEvictorTest {
         }
     }
 
+    @Test
+    public void testEvictionStatsMetaWithNoPattern() throws Exception {
+        try {
+            Configuration conf = cluster.getConf();
+            FileSystem fs = FileSystem.get(conf);
+            fs.delete(new Path("/"), true);
+            stream.clear();
+
+            Pair<List<String>, List<String>> pair = createTestData("/data");
+            createDir("/stats");
+            createDir("/meta");
+            createTestData("/tmp");
+            final String storageUrl = cluster.getConf().get(HadoopClientFactory.FS_DEFAULT_NAME_KEY);
+            FeedEvictor.main(new String[] {
+                "-feedBasePath",
+                getFeedBasePath(LocationType.DATA, storageUrl) + "#"
+                    + getStatsOrMetaPath(LocationType.STATS, storageUrl)
+                    + "#" + getStatsOrMetaPath(LocationType.META, storageUrl)
+                    + "#" + getFeedBasePath(LocationType.TMP, storageUrl),
+                "-retentionType", "instance",
+                "-retentionLimit", "months(5)",
+                "-timeZone", "UTC",
+                "-frequency", "hourly",
+                "-logFile", conf.get(HadoopClientFactory.FS_DEFAULT_NAME_KEY)
+                + "/falcon/staging/feed/2012-01-01-04-00", "-falconFeedStorageType",
+                Storage.TYPE.FILESYSTEM.name(),
+            });
+
+            // should not throw exception
+            // stats and meta dir should not be deleted
+            Assert.assertTrue(isDirPresent("/stats"));
+            Assert.assertTrue(isDirPresent("/meta"));
+        } catch (Exception e) {
+            Assert.fail("Unknown exception", e);
+        }
+    }
+
 
     private Pair<List<String>, List<String>> createTestData(String locationType)
throws Exception {
         Configuration conf = cluster.getConf();
@@ -482,6 +519,12 @@ public class FeedEvictorTest {
         return Pair.of(inRange, outOfRange);
     }
 
+    private void createDir(String locationType) throws Exception {
+        Configuration conf = cluster.getConf();
+        FileSystem fs = FileSystem.get(conf);
+        touch(fs, locationType, false);
+    }
+
     private Pair<List<String>, List<String>> createTestData(String feed,
String mask,
                                                             int period, TimeUnit timeUnit,
                                                             String locationType) throws Exception
{
@@ -542,11 +585,21 @@ public class FeedEvictorTest {
         }
     }
 
+    private boolean isDirPresent(String path) throws Exception {
+        FileSystem fs = FileSystem.get(cluster.getConf());
+        return fs.exists(new Path(path));
+    }
+
     private String getFeedBasePath(LocationType locationType, String storageUrl) {
         return locationType.name() + "=" + storageUrl
                 + "/" + locationType.name().toLowerCase() + "/data/YYYY/feed3/dd/MM/?{MONTH}/more/?{HOUR}";
     }
 
+    private String getStatsOrMetaPath(LocationType locationType, String storageUrl) {
+        return locationType.name() + "=" + storageUrl
+                + "/" + locationType.name().toLowerCase();
+    }
+
     private static class InMemoryWriter extends PrintStream {
 
         private final StringBuffer buffer = new StringBuffer();


Mime
View raw message