Return-Path: X-Original-To: apmail-falcon-commits-archive@minotaur.apache.org Delivered-To: apmail-falcon-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DAB851948B for ; Wed, 27 Apr 2016 07:03:44 +0000 (UTC) Received: (qmail 80351 invoked by uid 500); 27 Apr 2016 07:03:44 -0000 Delivered-To: apmail-falcon-commits-archive@falcon.apache.org Received: (qmail 80319 invoked by uid 500); 27 Apr 2016 07:03:44 -0000 Mailing-List: contact commits-help@falcon.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@falcon.apache.org Delivered-To: mailing list commits@falcon.apache.org Received: (qmail 80310 invoked by uid 99); 27 Apr 2016 07:03:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 27 Apr 2016 07:03:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A5330DFC6F; Wed, 27 Apr 2016 07:03:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: balu@apache.org To: commits@falcon.apache.org Message-Id: <6acf6479e76b4c58b2f8337218b6afb9@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: falcon git commit: FALCON-1909 Catalog instance triage action fails with null pointer exception Date: Wed, 27 Apr 2016 07:03:44 +0000 (UTC) Repository: falcon Updated Branches: refs/heads/master 2945fa80b -> 123391d46 FALCON-1909 Catalog instance triage action fails with null pointer exception …ception. Author: bvellanki Reviewers: " Praveen Adlakha , yzheng-hortonworks , Venkat Ranganathan " Closes #117 from bvellanki/FALCON-1909 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/123391d4 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/123391d4 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/123391d4 Branch: refs/heads/master Commit: 123391d460b5c1ed30209d10618377da023fdb4b Parents: 2945fa8 Author: bvellanki Authored: Wed Apr 27 00:03:34 2016 -0700 Committer: bvellanki Committed: Wed Apr 27 00:03:34 2016 -0700 ---------------------------------------------------------------------- .../apache/falcon/entity/CatalogStorage.java | 21 +++++++++++++------- .../falcon/entity/CatalogStorageTest.java | 10 ++++++---- .../apache/falcon/catalog/CatalogStorageIT.java | 12 ++++------- 3 files changed, 24 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/123391d4/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java ---------------------------------------------------------------------- diff --git a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java index c5860c9..31feee8 100644 --- a/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java +++ b/common/src/main/java/org/apache/falcon/entity/CatalogStorage.java @@ -100,7 +100,7 @@ public class CatalogStorage extends Configured implements Storage { if (catalogUrl == null || catalogUrl.length() == 0) { throw new IllegalArgumentException("Catalog Registry URL cannot be null or empty"); } - + verifyAndUpdateConfiguration(getConf()); this.catalogUrl = catalogUrl; parseFeedUri(tableUri); @@ -177,6 +177,10 @@ public class CatalogStorage extends Configured implements Storage { * @throws URISyntaxException */ protected CatalogStorage(String uriTemplate) throws URISyntaxException { + this(uriTemplate, new Configuration()); + } + + protected CatalogStorage(String uriTemplate, Configuration conf) throws URISyntaxException { if (uriTemplate == null || uriTemplate.length() == 0) { throw new IllegalArgumentException("URI template cannot be null or empty"); } @@ -186,15 +190,10 @@ public class CatalogStorage extends Configured implements Storage { URI uri = new URI(processed); this.catalogUrl = uri.getScheme() + "://" + uri.getAuthority(); - + verifyAndUpdateConfiguration(conf); parseUriTemplate(uri); } - protected CatalogStorage(String uriTemplate, Configuration conf) throws URISyntaxException { - this(uriTemplate); - setConf(conf); - } - private void parseUriTemplate(URI uriTemplate) throws URISyntaxException { String path = uriTemplate.getPath(); String[] paths = path.split(OUTPUT_PATH_SEPARATOR); @@ -589,4 +588,12 @@ public class CatalogStorage extends Configured implements Storage { + ", partitions=" + partitions + '}'; } + + private void verifyAndUpdateConfiguration(Configuration conf) { + if (conf == null) { + setConf(new Configuration()); + } else { + setConf(conf); + } + } } http://git-wip-us.apache.org/repos/asf/falcon/blob/123391d4/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java ---------------------------------------------------------------------- diff --git a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java index 5d06431..d20c722 100644 --- a/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java +++ b/common/src/test/java/org/apache/falcon/entity/CatalogStorageTest.java @@ -36,6 +36,7 @@ public class CatalogStorageTest { String table = "catalog:clicksdb:clicks#ds=${YEAR}-${MONTH}-${DAY};region=us"; CatalogStorage storage = new CatalogStorage(CatalogStorage.CATALOG_URL, table); Assert.assertEquals(Storage.TYPE.TABLE, storage.getType()); + Assert.assertNotNull(storage.getConf()); } @Test @@ -160,7 +161,7 @@ public class CatalogStorageTest { String uriTemplate = "thrift://localhost:49083/clicksdb/clicks/ds=${YEAR}-${MONTH}-${DAY};region=us"; CatalogStorage table = new CatalogStorage(catalogUrl, tableUri); - + Assert.assertNotNull(table.getConf()); Assert.assertEquals(uriTemplate, table.getUriTemplate()); Assert.assertEquals(uriTemplate, table.getUriTemplate(LocationType.DATA)); Assert.assertEquals(table.getUriTemplate(), table.getUriTemplate(LocationType.DATA)); @@ -172,7 +173,7 @@ public class CatalogStorageTest { String uriTemplate = "${hcatNode}/clicksdb/clicks/ds=${YEAR}-${MONTH}-${DAY};region=us"; CatalogStorage table = new CatalogStorage(CatalogStorage.CATALOG_URL, tableUri); - + Assert.assertNotNull(table.getConf()); Assert.assertEquals(uriTemplate, table.getUriTemplate()); Assert.assertEquals(uriTemplate, table.getUriTemplate(LocationType.DATA)); Assert.assertEquals(table.getUriTemplate(), table.getUriTemplate(LocationType.DATA)); @@ -183,9 +184,9 @@ public class CatalogStorageTest { final String catalogUrl = "thrift://localhost:49083"; String tableUri = "catalog:clicksdb:clicks#ds=20130918;region=us"; String partitionFilter = "(ds='20130918';region='us')"; - CatalogStorage table = new CatalogStorage(catalogUrl, tableUri); Assert.assertEquals(table.toPartitionFilter(), partitionFilter); + Assert.assertNotNull(table.getConf()); } @Test @@ -193,9 +194,9 @@ public class CatalogStorageTest { final String catalogUrl = "thrift://localhost:49083"; String tableUri = "catalog:clicksdb:clicks#ds=20130918;region=us"; String partitionPath = "ds=20130918/region=us"; - CatalogStorage table = new CatalogStorage(catalogUrl, tableUri); Assert.assertEquals(table.toPartitionAsPath(), partitionPath); + Assert.assertNotNull(table.getConf()); } @Test @@ -209,5 +210,6 @@ public class CatalogStorageTest { Assert.assertEquals(1, storage.getPartitions().size()); Assert.assertEquals("2012-04-21-00", storage.getPartitionValue("ds")); Assert.assertTrue(storage.hasPartition("ds")); + Assert.assertNotNull(storage.getConf()); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/123391d4/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java b/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java index bf3b2ec..262f8ac 100644 --- a/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java +++ b/webapp/src/test/java/org/apache/falcon/catalog/CatalogStorageIT.java @@ -24,16 +24,15 @@ import org.apache.falcon.entity.v0.Frequency; import org.apache.falcon.entity.v0.cluster.Interface; import org.apache.falcon.entity.v0.cluster.Interfaces; import org.apache.falcon.entity.v0.cluster.Interfacetype; -import org.apache.falcon.entity.v0.feed.Feed; -import org.apache.falcon.entity.v0.feed.Clusters; -import org.apache.falcon.entity.v0.feed.Validity; import org.apache.falcon.entity.v0.feed.CatalogTable; import org.apache.falcon.entity.v0.feed.ClusterType; +import org.apache.falcon.entity.v0.feed.Clusters; +import org.apache.falcon.entity.v0.feed.Feed; import org.apache.falcon.entity.v0.feed.LocationType; +import org.apache.falcon.entity.v0.feed.Validity; import org.apache.falcon.resource.TestContext; import org.apache.falcon.security.CurrentUser; import org.apache.falcon.util.HiveTestUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.hive.hcatalog.api.HCatAddPartitionDesc; import org.apache.hive.hcatalog.api.HCatClient; import org.apache.hive.hcatalog.common.HCatException; @@ -42,13 +41,12 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; - import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.HashMap; import java.util.TimeZone; @@ -116,8 +114,6 @@ public class CatalogStorageIT { feed.setTable(table); storage = new CatalogStorage(cluster, table); - Configuration configuration = HiveCatalogService.createHiveConf(new Configuration(), storage.getCatalogUrl()); - storage.setConf(configuration); }