hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From odiache...@apache.org
Subject incubator-hawq git commit: HAWQ-546. Implemented call of pxf_get_object_fields for Hive on psql.
Date Thu, 31 Mar 2016 00:49:52 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 7d155bee0 -> f86ead491


HAWQ-546. Implemented call of pxf_get_object_fields for Hive on psql.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f86ead49
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f86ead49
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f86ead49

Branch: refs/heads/master
Commit: f86ead49195513e4876688e135c05d17727c7587
Parents: 7d155be
Author: Oleksandr Diachenko <odiachenko@pivotal.io>
Authored: Tue Mar 29 14:44:05 2016 -0700
Committer: Oleksandr Diachenko <odiachenko@pivotal.io>
Committed: Wed Mar 30 17:45:43 2016 -0700

----------------------------------------------------------------------
 src/bin/psql/describe.c                        | 143 ++++++++++++++++++++
 src/test/regress/input/hcatalog_lookup.source  |   4 +
 src/test/regress/output/hcatalog_lookup.source |   7 +
 3 files changed, 154 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f86ead49/src/bin/psql/describe.c
----------------------------------------------------------------------
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index 99718d2..4ba1e2d 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -42,11 +42,16 @@ static bool describeOneTSConfig(const char *oid, const char *nspname,
 static void printACLColumn(PQExpBuffer buf, const char *colname);
 static bool isGPDB(void);
 static bool isGPDB4200OrLater(void);
+static bool describePxfTable(const char *profile, const char *pattern, bool verbose);
+static void parsePxfPattern(const char *user_pattern, char **pattern);
 
 /* GPDB 3.2 used PG version 8.2.10, and we've moved the minor number up since then for each
release,  4.1 = 8.2.15 */
 /* Allow for a couple of future releases.  If the version isn't in this range, we are talking
to PostgreSQL, not GPDB */
 #define mightBeGPDB() (pset.sversion >= 80210 && pset.sversion < 80222)
 
+#define HiveProfileName "Hive"
+#define HcatalogSourceName "hcatalog"
+
 static bool isGPDB(void)
 {
 	static enum { gpdb_maybe, gpdb_yes, gpdb_no } talking_to_gpdb;
@@ -1152,6 +1157,24 @@ describeTableDetails(const char *pattern, bool verbose, bool showSystem)
 	PGresult   *res;
 	int			i;
 
+	//Hive hook in this method
+	if(pattern && strncmp(pattern, HcatalogSourceName, strlen(HcatalogSourceName)) ==
0)
+	{
+		char *pxf_pattern = NULL;
+		char *pattern_dup = strdup(pattern);
+		parsePxfPattern(pattern_dup, &pxf_pattern);
+		if (!pxf_pattern)
+		{
+			fprintf(stderr, _("Invalid pattern provided.\n"));
+			free(pattern_dup);
+			return false;
+		}
+
+		bool success = describePxfTable(HiveProfileName, pxf_pattern, verbose);
+		free(pattern_dup);
+		return success;
+	}
+
 	initPQExpBuffer(&buf);
 
 	printfPQExpBuffer(&buf,
@@ -4213,3 +4236,123 @@ printACLColumn(PQExpBuffer buf, const char *colname)
 						  "pg_catalog.array_to_string(%s, '\\n') AS \"%s\"",
 						  colname, gettext_noop("Access privileges"));
 }
+
+/*
+ * parsePxfPattern
+ *
+ * Splits user_pattern by "." and writes second part to pattern.
+ */
+static void
+parsePxfPattern(const char *user_pattern, char **pattern)
+{
+	strtok(user_pattern, ".");
+	*pattern = strtok(NULL, "/0");
+}
+
+/*
+ * describePxfTable
+ *
+ * Describes external PXF table.
+ */
+static bool
+describePxfTable(const char *profile, const char *pattern, bool verbose)
+{
+	PQExpBufferData buf;
+	PQExpBufferData title;
+	PGresult *res;
+	printQueryOpt myopt = pset.popt;
+	printTableContent cont;
+	int			cols = 0;
+	int			total_numrows = 0;
+	char	   *headers[2];
+	bool		printTableInitialized = false;
+
+	char *previous_path = NULL;
+	char *previous_itemname = NULL;
+
+	char *path;
+	char *itemname;
+	char *fieldname;
+	char *fieldtype;
+	int total_fields = 0; //needed to know how much memory allocate for current table
+
+	initPQExpBuffer(&buf);
+
+	printfPQExpBuffer(&buf, "SELECT t.*, COUNT() OVER(PARTITION BY path, itemname) as total_fields
FROM\n"
+			"pxf_get_item_fields('%s', '%s') t\n", profile, pattern);
+
+	res = PSQLexec(buf.data, false);
+	total_numrows = PQntuples(res);
+	termPQExpBuffer(&buf);
+	if (!res)
+		return false;
+
+	myopt.nullPrint = NULL;
+	myopt.title = _("List of Hive tables");
+	myopt.translate_header = true;
+
+	/* Header */
+	headers[0] = gettext_noop("Column");
+	headers[1] = gettext_noop("Type");
+	cols = 2;
+
+	for (int i = 0; i < total_numrows; i++)
+	{
+
+		path = PQgetvalue(res, i, 0);
+		itemname = PQgetvalue(res, i, 1);
+		fieldname = PQgetvalue(res, i, 2);
+		fieldtype = PQgetvalue(res, i, 3);
+		total_fields = PQgetvalue(res, i, 4);
+
+		/* First row for current table */
+		if (previous_itemname == NULL
+				|| strncmp(previous_itemname, itemname,
+						strlen(previous_itemname)) != 0
+				|| strncmp(previous_path, path,
+						strlen(previous_path)) != 0)
+		{
+
+			if (previous_itemname != NULL)
+				printTable(&cont, pset.queryFout, pset.logfile);
+
+			/* Do clean-up for previous tables if any */
+			if (printTableInitialized)
+			{
+				printTableCleanup(&cont);
+				termPQExpBuffer(&title);
+				printTableInitialized = false;
+			}
+
+			/* Initialize */
+			initPQExpBuffer(&title);
+			printfPQExpBuffer(&title, _("PXF %s Table \"%s.%s\""), profile, path, itemname);
+			printTableInit(&cont, &myopt, title.data, cols, total_fields);
+			printTableInitialized = true;
+
+			for (int j = 0; j < cols; j++)
+				printTableAddHeader(&cont, headers[j], true, 'l');
+		}
+
+		/* Column */
+		printTableAddCell(&cont, fieldname, false, false);
+
+		/* Type */
+		printTableAddCell(&cont, fieldtype, false, false);
+
+		previous_path = path;
+		previous_itemname = itemname;
+
+	}
+
+	printTable(&cont, pset.queryFout, pset.logfile);
+
+	if (printTableInitialized)
+	{
+		printTableCleanup(&cont);
+		termPQExpBuffer(&title);
+	}
+
+	PQclear(res);
+	return true;
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f86ead49/src/test/regress/input/hcatalog_lookup.source
----------------------------------------------------------------------
diff --git a/src/test/regress/input/hcatalog_lookup.source b/src/test/regress/input/hcatalog_lookup.source
index 3d7279f..239494a 100644
--- a/src/test/regress/input/hcatalog_lookup.source
+++ b/src/test/regress/input/hcatalog_lookup.source
@@ -7,11 +7,15 @@ SET hcatalog_enable = false;
 SELECT * from hcatalog.db.t;
 
 SELECT * FROM pxf_get_item_fields('Hive', '*');
+ \d hcatalog.*.*
 
 -- enable GUC
 SET hcatalog_enable = true;
 
 SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*');
+\d hcatalog.*abc*.*abc*
+\d hcatalog
+\d hcatalog.
 
 -- Create function to insert and scan in-memory data to pg_class
 CREATE OR REPLACE FUNCTION convert_to_hcatalog_schema(schemaName text) RETURNS text

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f86ead49/src/test/regress/output/hcatalog_lookup.source
----------------------------------------------------------------------
diff --git a/src/test/regress/output/hcatalog_lookup.source b/src/test/regress/output/hcatalog_lookup.source
index ed4a7ef..533603a 100644
--- a/src/test/regress/output/hcatalog_lookup.source
+++ b/src/test/regress/output/hcatalog_lookup.source
@@ -9,6 +9,8 @@ LINE 1: SELECT * from hcatalog.db.t;
                       ^
 SELECT * FROM pxf_get_item_fields('Hive', '*');
 ERROR:  HCatalog querying is not enabled
+\d hcatalog.*.*
+ERROR:  HCatalog querying is not enabled
 -- enable GUC
 SET hcatalog_enable = true;
 SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*');
@@ -16,6 +18,11 @@ SELECT * FROM pxf_get_item_fields('Hive', '*abc*abc*');
 ------+----------+-----------+-----------
 (0 rows)
 
+\d hcatalog.*abc*.*abc*
+\d hcatalog
+Invalid pattern provided.
+\d hcatalog.
+Invalid pattern provided.
 -- Create function to insert and scan in-memory data to pg_class
 CREATE OR REPLACE FUNCTION convert_to_hcatalog_schema(schemaName text) RETURNS text
   AS '@abs_builddir@/regress@DLSUFFIX@', 'convert_to_hcatalog_schema'


Mime
View raw message