hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From odiache...@apache.org
Subject [2/4] incubator-hawq git commit: HAWQ-1228. Hardcoded comma as delimiter.
Date Tue, 24 Jan 2017 02:32:40 GMT
HAWQ-1228. Hardcoded comma as delimiter.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8fe27032
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8fe27032
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8fe27032

Branch: refs/heads/HAWQ-1228
Commit: 8fe27032f1faf786544a5109e854d319f6e6a1c4
Parents: 0574e75
Author: Oleksandr Diachenko <odiachenko@pivotal.io>
Authored: Fri Jan 6 18:39:32 2017 -0800
Committer: Oleksandr Diachenko <odiachenko@pivotal.io>
Committed: Fri Jan 6 18:39:32 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hawq/pxf/api/OutputFormat.java   |  2 +-
 .../plugins/hive/HiveColumnarSerdeResolver.java |  2 +-
 .../pxf/plugins/hive/HiveMetadataFetcher.java   |  2 +-
 .../pxf/plugins/hive/HiveORCSerdeResolver.java  | 22 --------------------
 .../hawq/pxf/plugins/hive/HiveResolver.java     |  3 ---
 .../plugins/hive/HiveStringPassResolver.java    |  2 +-
 src/backend/catalog/external/externalmd.c       |  2 +-
 src/bin/gpfusion/gpbridgeapi.c                  |  4 ++--
 src/include/access/hd_work_mgr.h                |  2 ++
 9 files changed, 9 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
index 82a747f..230f9ff 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java
@@ -23,4 +23,4 @@ package org.apache.hawq.pxf.api;
 /**
  * PXF supported output formats: {@link #TEXT} and {@link #BINARY}
  */
-public enum OutputFormat {TEXT, BINARY, UNKNOWN}
+public enum OutputFormat {TEXT, BINARY}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
index 2bf39ff..8fd37e0 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java
@@ -241,6 +241,6 @@ public class HiveColumnarSerdeResolver extends HiveResolver {
 
     @Override
     void parseDelimiterChar(InputData input) {
-        delimiter = 1;
+        delimiter = 44; //,
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
index 3d04bc1..76e8e18 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java
@@ -162,7 +162,7 @@ public class HiveMetadataFetcher extends MetadataFetcher {
     }
 
     private OutputFormat getOutputFormat(String inputFormat) {
-        OutputFormat outputFormat = OutputFormat.UNKNOWN;
+        OutputFormat outputFormat = null;
         try {
             InputFormat<?, ?> fformat = HiveDataFragmenter.makeInputFormat(inputFormat,
jobConf);
             String profile = ProfileFactory.get(fformat);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
index 381c407..0c7a417 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java
@@ -44,7 +44,6 @@ import java.util.*;
  */
 public class HiveORCSerdeResolver extends HiveResolver {
     private static final Log LOG = LogFactory.getLog(HiveORCSerdeResolver.class);
-    private OrcSerde deserializer;
     private HiveUtilities.PXF_HIVE_SERDES serdeType;
 
     public HiveORCSerdeResolver(InputData input) throws Exception {
@@ -63,27 +62,6 @@ public class HiveORCSerdeResolver extends HiveResolver {
                 : input.getUserProperty("MAPKEY_DELIM");
     }
 
-    /**
-     * getFields returns a singleton list of OneField item.
-     * OneField item contains two fields: an integer representing the VARCHAR type and a
Java
-     * Object representing the field value.
-     */
-    //TODO: It's the same as in parent class
-    @Override
-    public List<OneField> getFields(OneRow onerow) throws Exception {
-
-        Object tuple = deserializer.deserialize((Writable) onerow.getData());
-        // Each Hive record is a Struct
-        StructObjectInspector soi = (StructObjectInspector) deserializer.getObjectInspector();
-        List<OneField> record = traverseStruct(tuple, soi, false);
-
-        //Add partition fields if any
-        record.addAll(getPartitionFields());
-
-        return record;
-
-    }
-
     /*
      * Get and init the deserializer for the records of this Hive data fragment.
      * Suppress Warnings added because deserializer.initialize is an abstract function that
is deprecated

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
index 55d7205..b0dc2fb 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java
@@ -75,12 +75,9 @@ public class HiveResolver extends Plugin implements ReadResolver {
     protected static final String COLLECTION_DELIM = ",";
     protected String collectionDelim;
     protected String mapkeyDelim;
-    //private SerDe deserializer;
     protected SerDe deserializer;
     private List<OneField> partitionFields;
-    //private String serdeClassName;
     protected String serdeClassName;
-    //private String propsString;
     protected String propsString;
     String partitionKeys;
     protected char delimiter;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java
b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java
index c7cfb36..8c91d47 100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java
@@ -93,6 +93,6 @@ public class HiveStringPassResolver extends HiveResolver {
     }
 
     void parseDelimiterChar(InputData input) {
-        this.delimiter = 1;
+        delimiter = 44; //,
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/src/backend/catalog/external/externalmd.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/external/externalmd.c b/src/backend/catalog/external/externalmd.c
index ccdbdd6..c6cfb69 100644
--- a/src/backend/catalog/external/externalmd.c
+++ b/src/backend/catalog/external/externalmd.c
@@ -659,7 +659,7 @@ static Datum GetFormatOptionsForProfile(const List *outputFormats)
 	initStringInfo(&formatStr);
 	if (list_length(outputFormats) == 1 && strcmp(lfirst(list_head(outputFormats)),"TEXT")
== 0)
 	{
-		appendStringInfo(&formatStr, "delimiter '\x01' null '\N' escape '\'");
+		appendStringInfo(&formatStr, "delimiter '\x2C' null '\\N' escape '\\'");
 	} else {
 		appendStringInfo(&formatStr, "formatter 'pxfwritable_import'");
 	}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/src/bin/gpfusion/gpbridgeapi.c
----------------------------------------------------------------------
diff --git a/src/bin/gpfusion/gpbridgeapi.c b/src/bin/gpfusion/gpbridgeapi.c
index c5c217c..358c858 100644
--- a/src/bin/gpfusion/gpbridgeapi.c
+++ b/src/bin/gpfusion/gpbridgeapi.c
@@ -564,7 +564,7 @@ void free_token_resources(PxfInputData *inputData)
 
 static void	assign_optimal_supported_profile(char *profile, char *fmttype, char **supportedProfile,
char **supportedFormat)
 {
-	if (fmttype_is_text(*fmttype) && ((strcmp(profile, "HiveText") == 0) || (strcmp(profile,
"HiveRc") == 0)))
+	if (fmttype_is_text(*fmttype) && ((strcmp(profile, HiveTextProfileName) == 0) ||
(strcmp(profile, HiveRCProfileName) == 0)))
 	{
 		*supportedFormat = "TEXT";
 		*supportedProfile = profile;
@@ -575,6 +575,6 @@ static void	assign_optimal_supported_profile(char *profile, char *fmttype,
char
 	} else
 	{
 		*supportedFormat = "GPDBWritable";
-		*supportedProfile = "Hive";
+		*supportedProfile = HiveProfileName;
 	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/src/include/access/hd_work_mgr.h
----------------------------------------------------------------------
diff --git a/src/include/access/hd_work_mgr.h b/src/include/access/hd_work_mgr.h
index cab8ca7..ea4c6ef 100644
--- a/src/include/access/hd_work_mgr.h
+++ b/src/include/access/hd_work_mgr.h
@@ -48,5 +48,7 @@ PxfFragmentStatsElem *get_pxf_fragments_statistics(char *uri, Relation rel);
 List *get_pxf_item_metadata(char *profile, char *pattern, Oid dboid);
 
 #define HiveProfileName "Hive"
+#define HiveTextProfileName "HiveText"
+#define HiveRCProfileName "HiveRC"
 
 #endif   /* HDWORKMGR_H */


Mime
View raw message