falcon-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject falcon git commit: FALCON-1909 Catalog instance triage action fails with null pointer exception
Date Wed, 27 Apr 2016 07:03:44 GMT
Repository: falcon
Updated Branches:
  refs/heads/master 2945fa80b -> 123391d46


FALCON-1909 Catalog instance triage action fails with null pointer exception

…ception.

Author: bvellanki <bvellanki@hortonworks.com>

Reviewers: " Praveen Adlakha <adlakha.praveen@gmail.com>, yzheng-hortonworks <yzheng@hortonworks.com>,
Venkat Ranganathan <venkat@hortonworks.com>"

Closes #117 from bvellanki/FALCON-1909


Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/123391d4
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/123391d4
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/123391d4

Branch: refs/heads/master
Commit: 123391d460b5c1ed30209d10618377da023fdb4b
Parents: 2945fa8
Author: bvellanki <bvellanki@hortonworks.com>
Authored: Wed Apr 27 00:03:34 2016 -0700
Committer: bvellanki <bvellanki@hortonworks.com>
Committed: Wed Apr 27 00:03:34 2016 -0700

----------------------------------------------------------------------
 .../apache/falcon/entity/CatalogStorage.java    | 21 +++++++++++++-------
 .../falcon/entity/CatalogStorageTest.java       | 10 ++++++----
 .../apache/falcon/catalog/CatalogStorageIT.java | 12 ++++-------
 3 files changed, 24 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/falcon/blob/123391d4/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
index c5860c9..31feee8 100644
--- a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
+++ b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java
@@ -100,7 +100,7 @@ public class CatalogStorage extends Configured implements Storage {
         if (catalogUrl == null || catalogUrl.length() == 0) {
             throw new IllegalArgumentException("Catalog Registry URL cannot be null or empty");
         }
-
+        verifyAndUpdateConfiguration(getConf());
         this.catalogUrl = catalogUrl;
 
         parseFeedUri(tableUri);
@@ -177,6 +177,10 @@ public class CatalogStorage extends Configured implements Storage {
      * @throws URISyntaxException
      */
     protected CatalogStorage(String uriTemplate) throws URISyntaxException {
+        this(uriTemplate, new Configuration());
+    }
+
+    protected CatalogStorage(String uriTemplate, Configuration conf) throws URISyntaxException
{
         if (uriTemplate == null || uriTemplate.length() == 0) {
             throw new IllegalArgumentException("URI template cannot be null or empty");
         }
@@ -186,15 +190,10 @@ public class CatalogStorage extends Configured implements Storage {
         URI uri = new URI(processed);
 
         this.catalogUrl = uri.getScheme() + "://" + uri.getAuthority();
-
+        verifyAndUpdateConfiguration(conf);
         parseUriTemplate(uri);
     }
 
-    protected CatalogStorage(String uriTemplate, Configuration conf) throws URISyntaxException
{
-        this(uriTemplate);
-        setConf(conf);
-    }
-
     private void parseUriTemplate(URI uriTemplate) throws URISyntaxException {
         String path = uriTemplate.getPath();
         String[] paths = path.split(OUTPUT_PATH_SEPARATOR);
@@ -589,4 +588,12 @@ public class CatalogStorage extends Configured implements Storage {
                 + ", partitions=" + partitions
                 + '}';
     }
+
+    private void verifyAndUpdateConfiguration(Configuration conf) {
+        if (conf == null) {
+            setConf(new Configuration());
+        } else {
+            setConf(conf);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/123391d4/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
index 5d06431..d20c722 100644
--- a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java
@@ -36,6 +36,7 @@ public class CatalogStorageTest {
         String table = "catalog:clicksdb:clicks#ds=${YEAR}-${MONTH}-${DAY};region=us";
         CatalogStorage storage = new CatalogStorage(CatalogStorage.CATALOG_URL, table);
         Assert.assertEquals(Storage.TYPE.TABLE, storage.getType());
+        Assert.assertNotNull(storage.getConf());
     }
 
     @Test
@@ -160,7 +161,7 @@ public class CatalogStorageTest {
         String uriTemplate = "thrift://localhost:49083/clicksdb/clicks/ds=${YEAR}-${MONTH}-${DAY};region=us";
 
         CatalogStorage table = new CatalogStorage(catalogUrl, tableUri);
-
+        Assert.assertNotNull(table.getConf());
         Assert.assertEquals(uriTemplate, table.getUriTemplate());
         Assert.assertEquals(uriTemplate, table.getUriTemplate(LocationType.DATA));
         Assert.assertEquals(table.getUriTemplate(), table.getUriTemplate(LocationType.DATA));
@@ -172,7 +173,7 @@ public class CatalogStorageTest {
         String uriTemplate = "${hcatNode}/clicksdb/clicks/ds=${YEAR}-${MONTH}-${DAY};region=us";
 
         CatalogStorage table = new CatalogStorage(CatalogStorage.CATALOG_URL, tableUri);
-
+        Assert.assertNotNull(table.getConf());
         Assert.assertEquals(uriTemplate, table.getUriTemplate());
         Assert.assertEquals(uriTemplate, table.getUriTemplate(LocationType.DATA));
         Assert.assertEquals(table.getUriTemplate(), table.getUriTemplate(LocationType.DATA));
@@ -183,9 +184,9 @@ public class CatalogStorageTest {
         final String catalogUrl = "thrift://localhost:49083";
         String tableUri = "catalog:clicksdb:clicks#ds=20130918;region=us";
         String partitionFilter = "(ds='20130918';region='us')";
-
         CatalogStorage table = new CatalogStorage(catalogUrl, tableUri);
         Assert.assertEquals(table.toPartitionFilter(), partitionFilter);
+        Assert.assertNotNull(table.getConf());
     }
 
     @Test
@@ -193,9 +194,9 @@ public class CatalogStorageTest {
         final String catalogUrl = "thrift://localhost:49083";
         String tableUri = "catalog:clicksdb:clicks#ds=20130918;region=us";
         String partitionPath = "ds=20130918/region=us";
-
         CatalogStorage table = new CatalogStorage(catalogUrl, tableUri);
         Assert.assertEquals(table.toPartitionAsPath(), partitionPath);
+        Assert.assertNotNull(table.getConf());
     }
 
     @Test
@@ -209,5 +210,6 @@ public class CatalogStorageTest {
         Assert.assertEquals(1, storage.getPartitions().size());
         Assert.assertEquals("2012-04-21-00", storage.getPartitionValue("ds"));
         Assert.assertTrue(storage.hasPartition("ds"));
+        Assert.assertNotNull(storage.getConf());
     }
 }

http://git-wip-us.apache.org/repos/asf/falcon/blob/123391d4/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java b/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java
index bf3b2ec..262f8ac 100644
--- a/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java
+++ b/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java
@@ -24,16 +24,15 @@ import org.apache.falcon.entity.v0.Frequency;
 import org.apache.falcon.entity.v0.cluster.Interface;
 import org.apache.falcon.entity.v0.cluster.Interfaces;
 import org.apache.falcon.entity.v0.cluster.Interfacetype;
-import org.apache.falcon.entity.v0.feed.Feed;
-import org.apache.falcon.entity.v0.feed.Clusters;
-import org.apache.falcon.entity.v0.feed.Validity;
 import org.apache.falcon.entity.v0.feed.CatalogTable;
 import org.apache.falcon.entity.v0.feed.ClusterType;
+import org.apache.falcon.entity.v0.feed.Clusters;
+import org.apache.falcon.entity.v0.feed.Feed;
 import org.apache.falcon.entity.v0.feed.LocationType;
+import org.apache.falcon.entity.v0.feed.Validity;
 import org.apache.falcon.resource.TestContext;
 import org.apache.falcon.security.CurrentUser;
 import org.apache.falcon.util.HiveTestUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hive.hcatalog.api.HCatAddPartitionDesc;
 import org.apache.hive.hcatalog.api.HCatClient;
 import org.apache.hive.hcatalog.common.HCatException;
@@ -42,13 +41,12 @@ import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
 
-
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.HashMap;
 import java.util.TimeZone;
 
 
@@ -116,8 +114,6 @@ public class CatalogStorageIT {
         feed.setTable(table);
 
         storage = new CatalogStorage(cluster, table);
-        Configuration configuration = HiveCatalogService.createHiveConf(new Configuration(),
storage.getCatalogUrl());
-        storage.setConf(configuration);
     }
 
 


Mime
View raw message