hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shiv...@apache.org
Subject incubator-hawq git commit: HAWQ-615. Handle incomptible tables with getMetadata PXF API
Date Mon, 04 Apr 2016 22:41:18 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/HAWQ-615 [created] 676396fa0


HAWQ-615. Handle incomptible tables with getMetadata PXF API


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/676396fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/676396fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/676396fa

Branch: refs/heads/HAWQ-615
Commit: 676396fa0c40bb600aadda6f1dc7e8763a0bdde8
Parents: 59ebfa7
Author: Shivram Mani <shivram.mani@gmail.com>
Authored: Mon Apr 4 15:41:08 2016 -0700
Committer: Shivram Mani <shivram.mani@gmail.com>
Committed: Mon Apr 4 15:41:08 2016 -0700

----------------------------------------------------------------------
 .../pxf/plugins/hive/HiveMetadataFetcher.java   |  33 +++++-
 .../plugins/hive/HiveMetadataFetcherTest.java   | 112 ++++++++++++++++++-
 2 files changed, 140 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/676396fa/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
index d228ec5..a3accfc 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
@@ -50,9 +50,21 @@ public class HiveMetadataFetcher extends MetadataFetcher {
         client = HiveUtilities.initHiveClient();
     }
 
+    /**
+     * Fetches metadata of hive tables corresponding to the given pattern
+     * For patterns matching more than one table, the tables are skipped.
+     * If the pattern correspond to exactly one table, throws an exception if
+     * the table type is not supported or contains unsupported field types.
+     * Supported HCatalog types: TINYINT,
+     * SMALLINT, INT, BIGINT, BOOLEAN, FLOAT, DOUBLE, STRING, BINARY, TIMESTAMP,
+     * DATE, DECIMAL, VARCHAR, CHAR.
+     *
+     * @param pattern pattern table/file name or pattern in the given source
+     */
     @Override
     public List<Metadata> getMetadata(String pattern) throws Exception {
 
+        boolean ignoreErrors = false;
         List<Metadata.Item> tblsDesc = HiveUtilities.extractTablesFromPattern(client,
pattern);
 
         if(tblsDesc == null || tblsDesc.isEmpty()) {
@@ -62,11 +74,24 @@ public class HiveMetadataFetcher extends MetadataFetcher {
 
         List<Metadata> metadataList = new ArrayList<Metadata>();
 
+        if(tblsDesc.size() > 1) {
+            ignoreErrors = true;
+        }
+
         for(Metadata.Item tblDesc: tblsDesc) {
-            Metadata metadata = new Metadata(tblDesc);
-            Table tbl = HiveUtilities.getHiveTable(client, tblDesc);
-            getSchema(tbl, metadata);
-            metadataList.add(metadata);
+            try {
+                Metadata metadata = new Metadata(tblDesc);
+                Table tbl = HiveUtilities.getHiveTable(client, tblDesc);
+                getSchema(tbl, metadata);
+                metadataList.add(metadata);
+            } catch (UnsupportedTypeException | UnsupportedOperationException e) {
+                if(ignoreErrors) {
+                    LOG.warn("Metadata fetch for " + tblDesc.toString() + " failed. " + e.getMessage());
+                    continue;
+                } else {
+                    throw e;
+                }
+            }
         }
 
         return metadataList;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/676396fa/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
index 4ddb486..1323eea 100644
--- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
+++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcherTest.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.*;
 import static org.mockito.Mockito.*;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.logging.Log;
@@ -137,7 +138,7 @@ public class HiveMetadataFetcherTest {
         hiveTable.setPartitionKeys(new ArrayList<FieldSchema>());
         when(hiveClient.getTable("default", tableName)).thenReturn(hiveTable);
 
-        // get metadata
+        // Get metadata
         metadataList = fetcher.getMetadata(tableName);
         Metadata metadata = metadataList.get(0);
 
@@ -154,6 +155,115 @@ public class HiveMetadataFetcherTest {
         assertEquals("int4", field.getType());
     }
 
+    @Test
+    public void getTableMetadataWithMultipleTables() throws Exception {
+        prepareConstruction();
+
+        fetcher = new HiveMetadataFetcher(inputData);
+
+        String tablepattern = "*";
+        String dbpattern = "*";
+        String dbname = "default";
+        String tablenamebase = "regulartable";
+        String pattern = dbpattern + "." + tablepattern;
+
+        List<String> dbNames = new ArrayList<String>(Arrays.asList(dbname));
+        List<String> tableNames = new ArrayList<String>();
+
+        // Prepare for tables
+        List<FieldSchema> fields = new ArrayList<FieldSchema>();
+        fields.add(new FieldSchema("field1", "string", null));
+        fields.add(new FieldSchema("field2", "int", null));
+        StorageDescriptor sd = new StorageDescriptor();
+        sd.setCols(fields);
+
+        // Mock hive tables returned from hive client
+        for(int index=1;index<=2;index++) {
+            String tableName = tablenamebase + index;
+            tableNames.add(tableName);;
+            Table hiveTable = new Table();
+            hiveTable.setTableType("MANAGED_TABLE");
+            hiveTable.setSd(sd);
+            hiveTable.setPartitionKeys(new ArrayList<FieldSchema>());
+            when(hiveClient.getTable(dbname, tableName)).thenReturn(hiveTable);
+        }
+
+        // Mock database and table names return from hive client
+        when(hiveClient.getDatabases(dbpattern)).thenReturn(dbNames);
+        when(hiveClient.getTables(dbname, tablepattern)).thenReturn(tableNames);
+
+        // Get metadata
+        metadataList = fetcher.getMetadata(pattern);
+        assertEquals(2, metadataList.size());
+
+        for(int index=1;index<=2;index++) {
+            Metadata metadata = metadataList.get(index-1);
+            assertEquals(dbname + "." + tablenamebase+index, metadata.getItem().toString());
+            List<Metadata.Field> resultFields = metadata.getFields();
+            assertNotNull(resultFields);
+            assertEquals(2, resultFields.size());
+            Metadata.Field field = resultFields.get(0);
+            assertEquals("field1", field.getName());
+            assertEquals("text", field.getType()); // converted type
+            field = resultFields.get(1);
+            assertEquals("field2", field.getName());
+            assertEquals("int4", field.getType());
+        }
+    }
+
+    @Test
+    public void getTableMetadataWithIncompatibleTables() throws Exception {
+        prepareConstruction();
+
+        fetcher = new HiveMetadataFetcher(inputData);
+
+        String tablepattern = "*";
+        String dbpattern = "*";
+        String dbname = "default";
+        String pattern = dbpattern + "." + tablepattern;
+
+        String tableName1 = "viewtable";
+        // mock hive table returned from hive client
+        Table hiveTable1 = new Table();
+        hiveTable1.setTableType("VIRTUAL_VIEW");
+        when(hiveClient.getTable(dbname, tableName1)).thenReturn(hiveTable1);
+
+        String tableName2 = "regulartable";
+        // mock hive table returned from hive client
+        List<FieldSchema> fields = new ArrayList<FieldSchema>();
+        fields.add(new FieldSchema("field1", "string", null));
+        fields.add(new FieldSchema("field2", "int", null));
+        StorageDescriptor sd = new StorageDescriptor();
+        sd.setCols(fields);
+        Table hiveTable2 = new Table();
+        hiveTable2.setTableType("MANAGED_TABLE");
+        hiveTable2.setSd(sd);
+        hiveTable2.setPartitionKeys(new ArrayList<FieldSchema>());
+        when(hiveClient.getTable(dbname, tableName2)).thenReturn(hiveTable2);
+
+        // Mock get databases and tables return from hive client
+        List<String> tableNames = new ArrayList<String>(Arrays.asList(tableName1,
tableName2));
+        List<String> dbNames = new ArrayList<String>(Arrays.asList(dbname));
+        when(hiveClient.getDatabases(dbpattern)).thenReturn(dbNames);
+        when(hiveClient.getTables(dbname, tablepattern)).thenReturn(tableNames);
+
+        // Get metadata
+        metadataList = fetcher.getMetadata(pattern);
+        assertEquals(1, metadataList.size());
+        Metadata metadata = metadataList.get(0);
+        assertEquals(dbname + "." + tableName2, metadata.getItem().toString());
+
+        List<Metadata.Field> resultFields = metadata.getFields();
+        assertNotNull(resultFields);
+        assertEquals(2, resultFields.size());
+        Metadata.Field field = resultFields.get(0);
+        assertEquals("field1", field.getName());
+        assertEquals("text", field.getType()); // converted type
+        field = resultFields.get(1);
+        assertEquals("field2", field.getName());
+        assertEquals("int4", field.getType());
+    }
+
     private void prepareConstruction() throws Exception {
         hiveConfiguration = mock(HiveConf.class);
         PowerMockito.whenNew(HiveConf.class).withNoArguments().thenReturn(hiveConfiguration);


Mime
View raw message