hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shiv...@apache.org
Subject incubator-hawq git commit: HAWQ-459. Introduced metadata class in pxf profile
Date Fri, 18 Mar 2016 00:02:21 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/HAWQ-459 8ac035d5e -> c94f38aaa


HAWQ-459. Introduced metadata class in pxf profile


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c94f38aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c94f38aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c94f38aa

Branch: refs/heads/HAWQ-459
Commit: c94f38aaa4d319f436e9bf969553b49bcc283b2d
Parents: 8ac035d
Author: Shivram Mani <shivram.mani@gmail.com>
Authored: Thu Mar 17 17:02:16 2016 -0700
Committer: Shivram Mani <shivram.mani@gmail.com>
Committed: Thu Mar 17 17:02:16 2016 -0700

----------------------------------------------------------------------
 .../org/apache/hawq/pxf/api/MetadataFetcher.java     | 12 ++++++++----
 .../org/apache/hawq/pxf/api/utilities/InputData.java | 12 ++++++++++++
 .../hawq/pxf/plugins/hive/HiveMetadataFetcher.java   |  5 +++--
 .../pxf/plugins/hive/HiveMetadataFetcherTest.java    | 13 +++++++------
 .../hawq/pxf/service/MetadataFetcherFactory.java     | 14 ++++++--------
 .../hawq/pxf/service/rest/MetadataResource.java      | 15 ++++++++++++++-
 .../hawq/pxf/service/utilities/ProtocolData.java     |  1 +
 .../src/main/resources/pxf-profiles-default.xml      |  3 +++
 8 files changed, 54 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c94f38aa/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/MetadataFetcher.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/MetadataFetcher.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/MetadataFetcher.java
index 8ba25dc..5dc56dc 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/MetadataFetcher.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/MetadataFetcher.java
@@ -2,6 +2,9 @@ package org.apache.hawq.pxf.api;
 
 import java.util.List;
 
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Plugin;
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -25,22 +28,23 @@ import java.util.List;
 /**
  * Abstract class that defines getting metadata of an item.
  */
-public abstract class MetadataFetcher {
+public abstract class MetadataFetcher extends Plugin {
     protected List<Metadata> metadata;
 
     /**
      * Constructs a MetadataFetcher.
      *
+     * @param metaData the input data
      */
-    public MetadataFetcher() {
-
+    public MetadataFetcher(InputData metaData) {
+        super(metaData);
     }
 
     /**
      * Gets a metadata of a given table
      *
      * @param pattern table/file name or pattern
-     * @return metadata of given item(s)
+     * @return metadata of all items corresponding to given pattern
      * @throws Exception if metadata information could not be retrieved
      */
     public abstract List<Metadata> getMetadata(String pattern) throws Exception;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c94f38aa/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
index cbec312..891dba8 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/InputData.java
@@ -46,6 +46,7 @@ public class InputData {
     protected String accessor;
     protected String resolver;
     protected String fragmenter;
+    protected String metadata;
     protected String remoteLogin;
     protected String remoteSecret;
     protected int dataFragment; /* should be deprecated */
@@ -96,6 +97,7 @@ public class InputData {
         this.accessor = copy.accessor;
         this.resolver = copy.resolver;
         this.fragmenter = copy.fragmenter;
+        this.metadata = copy.metadata;
         this.remoteLogin = copy.remoteLogin;
         this.remoteSecret = copy.remoteSecret;
         this.threadSafe = copy.threadSafe;
@@ -272,6 +274,16 @@ public class InputData {
     }
 
     /**
+     * Returns the ClassName for the java class that was defined as Metadata
+     * or null if no metadata was defined.
+     *
+     * @return class name for METADATA or null
+     */
+    public String getMetadata() {
+        return metadata;
+    }
+
+    /**
      * Returns the contents of pxf_remote_service_login set in Hawq. Should the
      * user set it to an empty string this function will return null.
      *

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c94f38aa/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
index 367b0a5..c035a55 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hawq.pxf.api.Metadata;
 import org.apache.hawq.pxf.api.MetadataFetcher;
 import org.apache.hawq.pxf.api.UnsupportedTypeException;
+import org.apache.hawq.pxf.api.utilities.InputData;
 import org.apache.hawq.pxf.plugins.hive.utilities.HiveUtilities;
 
 /**
@@ -42,8 +43,8 @@ public class HiveMetadataFetcher extends MetadataFetcher {
     private static final Log LOG = LogFactory.getLog(HiveMetadataFetcher.class);
     private HiveMetaStoreClient client;
 
-    public HiveMetadataFetcher() {
-        super();
+    public HiveMetadataFetcher(InputData md) {
+        super(md);
 
         // init hive metastore client connection.
         client = HiveUtilities.initHiveClient();

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c94f38aa/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
index 2cc8b38..4ddb486 100644
--- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
+++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
 import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hawq.pxf.api.utilities.InputData;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -50,7 +51,7 @@ import org.apache.hawq.pxf.plugins.hive.utilities.HiveUtilities;
 @SuppressStaticInitializationFor({"org.apache.hadoop.hive.metastore.api.MetaException",
 "org.apache.hawq.pxf.plugins.hive.utilities.HiveUtilities"}) // Prevents static inits
 public class HiveMetadataFetcherTest {
-
+    InputData inputData;
     Log LOG;
     HiveConf hiveConfiguration;
     HiveMetaStoreClient hiveClient;
@@ -66,7 +67,7 @@ public class HiveMetadataFetcherTest {
     @Test
     public void construction() throws Exception {
         prepareConstruction();
-        fetcher = new HiveMetadataFetcher();
+        fetcher = new HiveMetadataFetcher(inputData);
         PowerMockito.verifyNew(HiveMetaStoreClient.class).withArguments(hiveConfiguration);
     }
 
@@ -76,7 +77,7 @@ public class HiveMetadataFetcherTest {
         PowerMockito.whenNew(HiveMetaStoreClient.class).withArguments(hiveConfiguration).thenThrow(new
MetaException("which way to albuquerque"));
 
         try {
-            fetcher = new HiveMetadataFetcher();
+            fetcher = new HiveMetadataFetcher(inputData);
             fail("Expected a RuntimeException");
         } catch (RuntimeException ex) {
             assertEquals("Failed connecting to Hive MetaStore service: which way to albuquerque",
ex.getMessage());
@@ -86,7 +87,7 @@ public class HiveMetadataFetcherTest {
     @Test
     public void getTableMetadataInvalidTableName() throws Exception {
         prepareConstruction();
-        fetcher = new HiveMetadataFetcher();
+        fetcher = new HiveMetadataFetcher(inputData);
         String tableName = "t.r.o.u.b.l.e.m.a.k.e.r";
 
         try {
@@ -101,7 +102,7 @@ public class HiveMetadataFetcherTest {
     public void getTableMetadataView() throws Exception {
         prepareConstruction();
 
-        fetcher = new HiveMetadataFetcher();
+        fetcher = new HiveMetadataFetcher(inputData);
         String tableName = "cause";
 
         // mock hive table returned from hive client
@@ -121,7 +122,7 @@ public class HiveMetadataFetcherTest {
     public void getTableMetadata() throws Exception {
         prepareConstruction();
 
-        fetcher = new HiveMetadataFetcher();
+        fetcher = new HiveMetadataFetcher(inputData);
         String tableName = "cause";
 
         // mock hive table returned from hive client

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c94f38aa/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataFetcherFactory.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataFetcherFactory.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataFetcherFactory.java
index c61446a..77f5d06 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataFetcherFactory.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataFetcherFactory.java
@@ -21,6 +21,8 @@ package org.apache.hawq.pxf.service;
 
 
 import org.apache.hawq.pxf.api.MetadataFetcher;
+import org.apache.hawq.pxf.api.utilities.InputData;
+import org.apache.hawq.pxf.api.utilities.Utilities;
 
 /**
  * Factory class for creation of {@link MetadataFetcher} objects. 
@@ -31,13 +33,9 @@ public class MetadataFetcherFactory {
     /* TODO: This is a tempororary workaround.
      * The metadata class will be moved to the pxf-profile.xml in the future
      */
-    public static MetadataFetcher create(String profile) throws Exception {
-        String fetcherClass;
-        if ( "hcatalog".equals(profile) || "hive".equals(profile) ) {
-            fetcherClass = "org.apache.hawq.pxf.plugins.hive.HiveMetadataFetcher";
-        } else {
-            throw new IllegalArgumentException("Metadata access for profile " + profile +
" not supported");
-        }
-        return (MetadataFetcher) Class.forName(fetcherClass).newInstance();
+    public static MetadataFetcher create(InputData inputData) throws Exception {
+        String metadataFetcher = inputData.getMetadata();
+
+        return (MetadataFetcher) Utilities.createAnyInstance(InputData.class, metadataFetcher,
inputData);
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c94f38aa/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
index 44d3ef1..cb50479 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/rest/MetadataResource.java
@@ -21,6 +21,7 @@ package org.apache.hawq.pxf.service.rest;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 
 import javax.servlet.ServletContext;
 import javax.ws.rs.GET;
@@ -38,8 +39,11 @@ import org.apache.commons.logging.LogFactory;
 
 import org.apache.hawq.pxf.api.Metadata;
 import org.apache.hawq.pxf.api.MetadataFetcher;
+import org.apache.hawq.pxf.api.utilities.InputData;
 import org.apache.hawq.pxf.service.MetadataFetcherFactory;
 import org.apache.hawq.pxf.service.MetadataResponseFormatter;
+import org.apache.hawq.pxf.service.utilities.ProtocolData;
+import org.apache.hawq.pxf.service.utilities.SecuredHDFS;
 
 /**
  * Class enhances the API of the WEBHDFS REST server. Returns the metadata of a
@@ -85,8 +89,17 @@ public class MetadataResource extends RestResource {
         LOG.debug("getMetadata started");
         String jsonOutput;
         try {
+
+            // Convert headers into a regular map
+            Map<String, String> params = convertToCaseInsensitiveMap(headers.getRequestHeaders());
+
+            // Add profile and verify token
+            params.put("X-GP-PROFILE", profile.toLowerCase());
+            ProtocolData protData = new ProtocolData(params);
+            SecuredHDFS.verifyToken(protData, servletContext);
+
             // 1. start MetadataFetcher
-            MetadataFetcher metadataFetcher = MetadataFetcherFactory.create(profile.toLowerCase());
+            MetadataFetcher metadataFetcher = MetadataFetcherFactory.create(protData);
 
             // 2. get Metadata
             List<Metadata> metadata = metadataFetcher.getMetadata(pattern);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c94f38aa/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
index 6ab224a..50ce46a 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
@@ -95,6 +95,7 @@ public class ProtocolData extends InputData {
         accessor = getProperty("ACCESSOR");
         resolver = getProperty("RESOLVER");
         fragmenter = getOptionalProperty("FRAGMENTER");
+        metadata = getOptionalProperty("METADATA");
         dataSource = getProperty("DATA-DIR");
 
         /* Kerberos token information */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c94f38aa/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
index 4664345..e7afdce 100644
--- a/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
+++ b/pxf/pxf-service/src/main/resources/pxf-profiles-default.xml
@@ -49,6 +49,7 @@ under the License.
             <fragmenter>org.apache.hawq.pxf.plugins.hive.HiveDataFragmenter</fragmenter>
             <accessor>org.apache.hawq.pxf.plugins.hive.HiveAccessor</accessor>
             <resolver>org.apache.hawq.pxf.plugins.hive.HiveResolver</resolver>
+            <metadata>org.apache.hawq.pxf.plugins.hive.HiveMetadataFetcher</metadata>
         </plugins>
     </profile>
     <profile>
@@ -62,6 +63,7 @@ under the License.
             <fragmenter>org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter</fragmenter>
             <accessor>org.apache.hawq.pxf.plugins.hive.HiveRCFileAccessor</accessor>
             <resolver>org.apache.hawq.pxf.plugins.hive.HiveColumnarSerdeResolver</resolver>
+            <metadata>org.apache.hawq.pxf.plugins.hive.HiveMetadataFetcher</metadata>
         </plugins>
     </profile>
     <profile>
@@ -74,6 +76,7 @@ under the License.
             <fragmenter>org.apache.hawq.pxf.plugins.hive.HiveInputFormatFragmenter</fragmenter>
             <accessor>org.apache.hawq.pxf.plugins.hive.HiveLineBreakAccessor</accessor>
             <resolver>org.apache.hawq.pxf.plugins.hive.HiveStringPassResolver</resolver>
+            <metadata>org.apache.hawq.pxf.plugins.hive.HiveMetadataFetcher</metadata>
         </plugins>
     </profile>
     <profile>


Mime
View raw message