hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nh...@apache.org
Subject [1/3] incubator-hawq git commit: HAWQ-44. Advanced statistics for PXF tables.
Date Fri, 20 Nov 2015 21:50:39 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 4dbb3479f -> 81385f09f


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/backend/access/external/test/pxfanalyze_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/pxfanalyze_test.c b/src/backend/access/external/test/pxfanalyze_test.c
new file mode 100644
index 0000000..a70c470
--- /dev/null
+++ b/src/backend/access/external/test/pxfanalyze_test.c
@@ -0,0 +1,171 @@
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include "cmockery.h"
+
+#include "c.h"
+#include "../pxfanalyze.c"
+#include "catalog/pg_exttable.h"
+
+
+static void runTest__calculateSamplingRatio(float4 relTuples, float4 relFrags, float4 requestedSampleSize,
+		int maxFrags, float4 expectedResult);
+static void runTest__createPxfSampleStmt(float4 pxf_sample_ratio,
+		const char* expectedRatio,
+		char fmtcode,
+		const char* expectedFmt,
+		const char* fmtopts,
+		const char* expectedFmtopts,
+		int rejectLimit);
+
+
+static void runTest__calculateSamplingRatio(float4 relTuples, float4 relFrags, float4 requestedSampleSize,
+		int maxFrags, float4 expectedResult)
+{
+	int pxf_stat_max_fragments_orig = pxf_stat_max_fragments;
+
+	pxf_stat_max_fragments = maxFrags;
+
+	float4 result = calculateSamplingRatio(relTuples, relFrags, requestedSampleSize);
+
+	pxf_stat_max_fragments = pxf_stat_max_fragments_orig;
+
+	assert_true(fabs(expectedResult - result) <= 0.00001);
+}
+
+void
+test__calculateSamplingRatio__relFragsLTmaxFrags(void **state)
+{
+	/*
+	 * original ratio: 20,000/100,000 = 0.20
+	 */
+	runTest__calculateSamplingRatio(100000, 1000, 20000, 1500, 0.2);
+}
+
+void
+test__calculateSamplingRatio__relFragsGTmaxFrags(void **state)
+{
+	/*
+	 * original ratio: 20,000/100,000 = 0.20
+	 * corrected ratio: 0.20*(1000/900) ~ 0.22
+	 */
+	runTest__calculateSamplingRatio(100000, 1000, 20000, 900, 0.222223);
+}
+
+void
+test__calculateSamplingRatio__ratioGT1(void **state)
+{
+	/*
+	 * original ratio: 20,000/100,000 = 0.20
+	 * corrected ratio: 0.20*(1000/100)=2.0 -> 1.0
+	 */
+	runTest__calculateSamplingRatio(100000, 1000, 20000, 100, 1.0);
+}
+
+void
+test__calculateSamplingRatio__ratioTooLow(void **state)
+{
+	/*
+	 * original ratio: 2,000/100,000,000 = 0.00002
+	 * corrected ratio: 0.0001
+	 */
+	runTest__calculateSamplingRatio(100000000, 1000, 2000, 1000, 0.0001);
+}
+
+static void runTest__createPxfSampleStmt(float4 pxf_sample_ratio,
+		const char* expectedRatio,
+		char fmtcode,
+		const char* expectedFmt,
+		const char* fmtopts,
+		const char* expectedFmtopts,
+		int rejectLimit)
+{
+		/* input */
+		Oid relationOid = 13;
+		const char* schemaName = "orig_schema";
+		const char* tableName = "orig_table";
+		const char* sampleSchemaName = "sample_schema";
+		const char* pxfSampleTable = "pxf_sample_table";
+
+		int pxf_max_fragments = 1000;
+
+		const char* location = "the_table-s_location";
+
+		Value* locationValue = (Value *) palloc0(sizeof(Value));
+		locationValue->type = T_String;
+		locationValue->val.str = location;
+
+		ExtTableEntry *extTable = (ExtTableEntry *) palloc0(sizeof(ExtTableEntry));
+		extTable->encoding = 6;
+		extTable->fmtcode = fmtcode;
+		extTable->fmtopts = fmtopts;
+		extTable->rejectlimit = rejectLimit;
+		extTable->locations = lappend(extTable->locations, locationValue);
+
+		/* get fake external table details */
+		expect_value(GetExtTableEntry, relid, relationOid);
+		will_return(GetExtTableEntry, extTable);
+
+		char* expectedResult = palloc0(1024);
+		sprintf(expectedResult,
+				"CREATE EXTERNAL TABLE %s.%s (LIKE %s.%s) "
+				"LOCATION(E'%s&STATS-SAMPLE-RATIO=%s&STATS-MAX-FRAGMENTS=%d') "
+				"FORMAT '%s' (%s) "
+				"ENCODING 'UTF8' "
+				"%s",
+				sampleSchemaName, pxfSampleTable, schemaName, tableName,
+				location, expectedRatio, pxf_max_fragments,
+				expectedFmt, expectedFmtopts,
+				(rejectLimit != -1) ? "SEGMENT REJECT LIMIT 25 PERCENT " : "");
+
+		char* result = createPxfSampleStmt(relationOid, schemaName, tableName, sampleSchemaName,
pxfSampleTable,
+				pxf_sample_ratio, pxf_max_fragments);
+
+		assert_string_equal(expectedResult, result);
+
+		pfree(locationValue);
+		pfree(extTable);
+		pfree(expectedResult);
+		pfree(result);
+}
+
+void
+test__createPxfSampleStmt__textFormat(void **state)
+{
+	const char* fmtopts = "delimiter ',' null '\\N' escape '\\'";
+	const char* expectedFmtopts = "delimiter E',' null E'\\\\N' escape E'\\\\'";
+	runTest__createPxfSampleStmt(0.12, "0.1200", 't', "text", fmtopts, expectedFmtopts, 30);
+}
+
+void
+test__createPxfSampleStmt__customFormatNoRejectLimit(void **state)
+{
+	const char* fmtopts = "formatter 'pxfwritable_import'";
+	const char* expectedFmtopts = "formatter = 'pxfwritable_import'";
+	runTest__createPxfSampleStmt(0.5555555, "0.5556", 'b', "custom", fmtopts, expectedFmtopts,
-1);
+}
+
+void
+test__createPxfSampleStmt__csvFormatUnprintableOptions(void **state)
+{
+	const char* fmtopts = "delimiter '\x01' null '\\N' escape '\x02\x03'";
+	const char* expectedFmtopts = "delimiter E'\\x01' null E'\\\\N' escape E'\\x02\\x03'";
+	runTest__createPxfSampleStmt(0.003, "0.0030", 'c', "csv", fmtopts, expectedFmtopts, 100);
+}
+
+int
+main(int argc, char* argv[])
+{
+	cmockery_parse_arguments(argc, argv);
+
+	const UnitTest tests[] = {
+			unit_test(test__calculateSamplingRatio__relFragsLTmaxFrags),
+			unit_test(test__calculateSamplingRatio__relFragsGTmaxFrags),
+			unit_test(test__calculateSamplingRatio__ratioGT1),
+			unit_test(test__calculateSamplingRatio__ratioTooLow),
+			unit_test(test__createPxfSampleStmt__textFormat),
+			unit_test(test__createPxfSampleStmt__customFormatNoRejectLimit),
+			unit_test(test__createPxfSampleStmt__csvFormatUnprintableOptions)
+	};
+	return run_tests(tests);
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/backend/access/external/test/pxfmasterapi_test.c
----------------------------------------------------------------------
diff --git a/src/backend/access/external/test/pxfmasterapi_test.c b/src/backend/access/external/test/pxfmasterapi_test.c
index dbabb5b..816b277 100644
--- a/src/backend/access/external/test/pxfmasterapi_test.c
+++ b/src/backend/access/external/test/pxfmasterapi_test.c
@@ -234,6 +234,24 @@ test__rest_request__callRestHASuccessFromTheFirstCall(void **state)
 	pfree(client_context);
 }
 
+void
+test__normalize_size(void **state)
+{
+	float4 result = normalize_size(10000000, "B");
+	assert_int_equal(result, 10000000);
+
+	result = normalize_size(10000000, "KB");
+	assert_int_equal(result, 10240000000);
+
+	result = normalize_size(500, "MB");
+	assert_int_equal(result, 524288000);
+
+	result = normalize_size(10, "GB");
+	assert_int_equal(result, 10737418240);
+
+	result = normalize_size(10000, "TB");
+	assert_int_equal(result, 10995116277760000);
+}
 
 int 
 main(int argc, char *argv[]) 
@@ -244,7 +262,8 @@ main(int argc, char *argv[])
 		    unit_test(test__rest_request__callRestThrowsNoHA),
 		    unit_test(test__rest_request__callRestThrowsHAFirstTime),
 		    unit_test(test__rest_request__callRestThrowsHASecondTime),
-		    unit_test(test__rest_request__callRestHASuccessFromTheFirstCall)
+		    unit_test(test__rest_request__callRestHASuccessFromTheFirstCall),
+			unit_test(test__normalize_size)
 	};
 	return run_tests(tests);
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/backend/commands/analyze.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 67fc8d4..fcbedb0 100644
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -28,6 +28,7 @@
 #include "access/pxfuriparser.h"
 #include "access/heapam.h"
 #include "access/hd_work_mgr.h"
+#include "access/pxfanalyze.h"
 #include "catalog/catquery.h"
 #include "catalog/heap.h"
 #include "access/transam.h"
@@ -140,13 +141,12 @@ static bool hasMaxDefined(Oid relationOid, const char *attributeName);
 
 /* Sampling related */
 static float4 estimateSampleSize(Oid relationOid, const char *attributeName, float4 relTuples);
-static char* temporarySampleTableName(Oid relationOid);
-static Oid buildSampleTable(Oid relationOid, 
+static Oid buildSampleTable(Oid relationOid,
+		char* sampleTableName,
 		List *lAttributeNames, 
 		float4	relTuples,
 		float4 	requestedSampleSize, 
 		float4 *sampleTableRelTuples);
-static void dropSampleTable(Oid sampleTableOid);
 
 /* Attribute statistics computation */
 static int4 numberOfMCVEntries(Oid relationOid, const char *attributeName);
@@ -198,25 +198,6 @@ static void updateAttributeStatisticsInCatalog(Oid relationOid, const
char *attr
 		AttributeStatistics *stats);
 static void updateReltuplesRelpagesInCatalog(Oid relationOid, float4 relTuples, float4 relPages);
 
-/* Convenience */
-static ArrayType * SPIResultToArray(int resultAttributeNumber, MemoryContext allocationContext);
-
-/* spi execution helpers */
-typedef void (*spiCallback)(void *clientDataOut);
-static void spiExecuteWithCallback(const char *src, bool read_only, long tcount,
-           spiCallback callbackFn, void *clientData);
-
-typedef struct
-{
-    int numColumns;
-    MemoryContext memoryContext;
-    ArrayType ** output;
-} EachResultColumnAsArraySpec;
-
-static void spiCallback_getEachResultColumnAsArray(void *clientData);
-static void spiCallback_getProcessedAsFloat4(void *clientData);
-static void spiCallback_getSingleResultRowColumnAsFloat4(void *clientData);
-
 /**
  * Extern stuff.
  */
@@ -604,9 +585,9 @@ void analyzeStmt(VacuumStmt *stmt, List *relids)
 						FaultInjector_InjectFaultIfSet(
 								AnalyzeSubxactError,
 								DDLNotSpecified,
-								"",  // databaseName
-								""); // tableName
-#endif // FAULT_INJECTOR
+								"",  /* databaseName */
+								""); /* tableName */
+#endif /* FAULT_INJECTOR */
 
 						ReleaseCurrentSubTransaction();
 						MemoryContextSwitchTo(oldcontext);
@@ -883,7 +864,7 @@ static List *analyzableAttributes(Relation candidateRelation)
 				|| attr->atttypid == UNKNOWNOID))
 		{
 			char	*attName = NULL;
-			attName = pstrdup(NameStr(attr->attname)); //needs to be pfree'd by caller
+			attName = pstrdup(NameStr(attr->attname)); /* needs to be pfree'd by caller */
 			Assert(attName);
 			lAttNames = lappend(lAttNames, (void *) attName);
 		}
@@ -932,16 +913,7 @@ static void analyzeRelation(Relation relation, List *lAttributeNames,
bool rooto
 	}
 	else
 	{
-		initStringInfo(&err_msg);
-		gp_statistics_estimate_reltuples_relpages_external_pxf(relation, &location, &estimatedRelTuples,
&estimatedRelPages, &err_msg);
-		if (err_msg.len > 0)
-		{
-			ereport(WARNING,
-					(errmsg("skipping \"%s\" --- error returned: %s",
-							RelationGetRelationName(relation),
-							err_msg.data)));
-		}
-		pfree(err_msg.data);
+		analyzePxfEstimateReltuplesRelpages(relation, &location, &estimatedRelTuples, &estimatedRelPages);
 	}
 	pfree(location.data);
 	
@@ -987,15 +959,6 @@ static void analyzeRelation(Relation relation, List *lAttributeNames,
bool rooto
 	pgstat_report_analyze(relation, estimatedRelTuples, 0 /*totaldeadrows*/);
 	
 	/**
-	 * For an external PXF table, the next steps are irrelevant - it's time to leave
-	 */
-	if (isExternalPxfReadOnly)
-	{
-		elog(elevel, "ANALYZE on PXF table %s computes only reltuples and relpages.", RelationGetRelationName(relation));
-		return;
-	}
-
-	/**
 	 * Does the relation have any rows. If not, no point analyzing columns.
 	 */
 	if (estimatedRelTuples < 1.0)
@@ -1053,9 +1016,13 @@ static void analyzeRelation(Relation relation, List *lAttributeNames,
bool rooto
 	 * Determine if a sample table needs to be created. If reltuples is very small,
 	 * then, we'd rather work off the entire table. Also, if the sample required is
 	 * the size of the table, then we'd rather work off the entire table.
+	 *
+	 * In case of PXF table, we always need a sample table because the various calculations
+	 * should be done locally in HAWQ and not by retrieving the data again and again.
 	 */
-	if (estimatedRelTuples <= gp_statistics_sampling_threshold 
-			|| minSampleTableSize >= estimatedRelTuples) /* maybe this should be K% of reltuples
or something? */
+	if (!isExternalPxfReadOnly &&
+			(estimatedRelTuples <= gp_statistics_sampling_threshold
+			|| minSampleTableSize >= estimatedRelTuples)) /* maybe this should be K% of reltuples
or something? */
 	{
 		sampleTableRequired = false;
 	}
@@ -1065,11 +1032,33 @@ static void analyzeRelation(Relation relation, List *lAttributeNames,
bool rooto
 	 */
 	if (sampleTableRequired)
 	{
-		elog(elevel, "ANALYZE building sample table of size %.0f on table %s because it has too
many rows.", minSampleTableSize, RelationGetRelationName(relation));
-		sampleTableOid = buildSampleTable(relationOid, lAttributeNames, estimatedRelTuples, minSampleTableSize,
&sampleTableRelTuples);
-		
+		char * sampleTableName = temporarySampleTableName(relationOid, "pg_analyze"); /* must be
pfreed */
+
+		elog(elevel, "ANALYZE building sample table of size %.0f on table %s because %s.",
+				minSampleTableSize, RelationGetRelationName(relation),
+				isExternalPxfReadOnly ? "it's a PXF table" : "it has too many rows");
+
+		if (isExternalPxfReadOnly)
+		{
+			sampleTableOid = buildPxfSampleTable(relationOid, sampleTableName, lAttributeNames,
+					estimatedRelTuples, estimatedRelPages, minSampleTableSize,
+					&sampleTableRelTuples);
+		}
+		else
+		{
+			sampleTableOid = buildSampleTable(relationOid, sampleTableName, lAttributeNames,
+					estimatedRelTuples, minSampleTableSize, &sampleTableRelTuples);
+		}
+		/*
+		 * Update the sample table's reltuples, relpages. Without these, the queries to the sample
table would call cdbRelsize which can be an expensive call.
+		 * We know the number of tuples in the sample table, but don't have the information about
the number of pages. We set it to 2 arbitrarily.
+		 */
+		updateReltuplesRelpagesInCatalog(sampleTableOid, sampleTableRelTuples, 2);
+
 		/* We must have a non-empty sample table */
-		Assert(sampleTableRelTuples > 0.0);	
+		Assert(sampleTableRelTuples > 0.0);
+
+		pfree((void *) sampleTableName);
 	}
 	
 	/**
@@ -1093,7 +1082,7 @@ static void analyzeRelation(Relation relation, List *lAttributeNames,
bool rooto
 	if (sampleTableRequired)
 	{
 		elog(elevel, "ANALYZE dropping sample table");
-		dropSampleTable(sampleTableOid);
+		dropSampleTable(sampleTableOid, false);
 	}
 	
 	return;
@@ -1104,14 +1093,14 @@ static void analyzeRelation(Relation relation, List *lAttributeNames,
bool rooto
  * This is not super random. However, this should be sufficient for our purpose.
  * Input:
  * 	relationOid 	- relation
- * 	backendId	- pid of the backend.
+ * 	prefix			- sample name prefix
  * Output:
  * 	sample table name. This must be pfree'd by the caller.
  */
-static char* temporarySampleTableName(Oid relationOid)
+char* temporarySampleTableName(Oid relationOid, char* prefix)
 {
 	char tmpname[NAMEDATALEN];
-	snprintf(tmpname, NAMEDATALEN, "pg_analyze_%u_%i", relationOid, MyBackendId);
+	snprintf(tmpname, NAMEDATALEN, "%s_%u_%i", prefix, relationOid, MyBackendId);
 	return pstrdup(tmpname);
 }
 
@@ -1217,18 +1206,18 @@ static float4 estimateSampleSize(Oid relationOid, const char *attributeName,
flo
  * 	read_only - is it a read-only call?
  * 	tcount - execution tuple-count limit, or 0 for none
  * 	callbackFn - callback function to be executed once SPI is done.
- * 	clientData - argument to call back function (usually pointer to data-structure 
+ * 	clientData - argument to call back function (usually pointer to data-structure
  * 				that the callback function populates).
- * 
+ *
  */
-static void spiExecuteWithCallback(
+void spiExecuteWithCallback(
 		const char *src,
 		bool read_only,
 		long tcount,
 		spiCallback callbackFn,
 		void *clientData)
 {
-	bool connected = false;
+	volatile bool connected = false; /* needs to be volatile when accessed by PG_CATCH */
 	int ret = 0;
 
 	PG_TRY();
@@ -1236,11 +1225,11 @@ static void spiExecuteWithCallback(
 		if (SPI_OK_CONNECT != SPI_connect())
 		{
 			ereport(ERROR, (errcode(ERRCODE_CDB_INTERNAL_ERROR),
-					errmsg("Unable to connect to execute internal query.")));
+					errmsg("Unable to connect to execute internal query: %s.", src)));
 		}
 		connected = true;
 
-		elog(elevel, "Executing SQL: %s", src);
+		elog(DEBUG2, "Executing SQL: %s", src);
 		
 		/* Do the query. */
 		ret = SPI_execute(src, read_only, tcount);
@@ -1251,13 +1240,17 @@ static void spiExecuteWithCallback(
 			callbackFn(clientData);
 		}
 		connected = false;
-		SPI_finish();
+		int res = SPI_finish();
+		elog(DEBUG5, "finish SPI %s, res %d, ret %d", src, res, ret);
 	}
 	/* Clean up in case of error. */
 	PG_CATCH();
 	{
 		if (connected)
-			SPI_finish();
+		{
+			int res = SPI_finish();
+			elog(DEBUG5, "finish SPI %s after error, res %d, ret %d", src, res, ret);
+		}
 
 		/* Carry on with error handling. */
 		PG_RE_THROW();
@@ -1269,15 +1262,15 @@ static void spiExecuteWithCallback(
  * A callback function for use with spiExecuteWithCallback.  Asserts that exactly one row
was returned.
  *  Gets the row's first column as a float, using 0.0 if the value is null
  */
-static void spiCallback_getSingleResultRowColumnAsFloat4(void *clientData)
+void spiCallback_getSingleResultRowColumnAsFloat4(void *clientData)
 {
 	Datum datum_f;
 	bool isnull = false;
 	float4 *out = (float4*) clientData;
 
-    Assert(SPI_tuptable != NULL); // must have result
-    Assert(SPI_processed == 1); //we expect only one tuple.
-	Assert(SPI_tuptable->tupdesc->attrs[0]->atttypid == FLOAT4OID); // must be float4
+    Assert(SPI_tuptable != NULL); /* must have result */
+    Assert(SPI_processed == 1); /* we expect only one tuple. */
+	Assert(SPI_tuptable->tupdesc->attrs[0]->atttypid == FLOAT4OID); /* must be float4
*/
 
 	datum_f = heap_getattr(SPI_tuptable->vals[0], 1, SPI_tuptable->tupdesc, &isnull);
 
@@ -1295,7 +1288,7 @@ static void spiCallback_getSingleResultRowColumnAsFloat4(void *clientData)
  * A callback function for use with spiExecuteWithCallback.  Copies the SPI_processed value
into
  *    *clientDataOut, treating it as a float4 pointer.
  */
-static void spiCallback_getProcessedAsFloat4(void *clientData)
+void spiCallback_getProcessedAsFloat4(void *clientData)
 {
     float4 *out = (float4*) clientData;
     *out = (float4)SPI_processed;
@@ -1306,7 +1299,7 @@ static void spiCallback_getProcessedAsFloat4(void *clientData)
  *   The number of arrays, the memory context for them, and the output location are determined
by
  *   treating *clientData as a EachResultColumnAsArraySpec and using the values there
  */
-static void spiCallback_getEachResultColumnAsArray(void *clientData)
+void spiCallback_getEachResultColumnAsArray(void *clientData)
 {
     EachResultColumnAsArraySpec * spec = (EachResultColumnAsArraySpec*) clientData;
     int i;
@@ -1334,14 +1327,16 @@ static void spiCallback_getEachResultColumnAsArray(void *clientData)
  * 
  * Input:
  * 	relationOid 	- relation to be sampled
+ * 	sampleTableName - sample table name, moderately unique
  * 	lAttributeNames - attributes to be included in the sample
  * 	relTuples		- estimated size of relation
  * 	requestedSampleSize - as determined by attribute statistics requirements.
- * 	sampleLimit		- limit on size of the sample.
+ * 	sampleTableRelTuples    - limit on size of the sample.
  * Output:
  * 	sampleTableRelTuples - number of tuples in the sample table created.
  */
 static Oid buildSampleTable(Oid relationOid, 
+		char* sampleTableName,
 		List *lAttributeNames, 
 		float4	relTuples,
 		float4 	requestedSampleSize, 
@@ -1354,7 +1349,6 @@ static Oid buildSampleTable(Oid relationOid,
 	const char *schemaName = NULL;
 	const char *tableName = NULL;
 	char	*sampleSchemaName = pstrdup("pg_temp"); 
-	char 	*sampleTableName = NULL;
 	Oid			sampleTableOid = InvalidOid;
 	float4		randomThreshold = 0.0;
 	RangeVar 	*rangeVar = NULL;
@@ -1364,11 +1358,11 @@ static Oid buildSampleTable(Oid relationOid,
 	
 	randomThreshold = requestedSampleSize / relTuples;
 	
-	schemaName = get_namespace_name(get_rel_namespace(relationOid)); //must be pfreed
-	tableName = get_rel_name(relationOid); //must be pfreed
-	sampleTableName = temporarySampleTableName(relationOid); // must be pfreed 
+	schemaName = get_namespace_name(get_rel_namespace(relationOid)); /* must be pfreed */
+	tableName = get_rel_name(relationOid); /* must be pfreed */
 
 	initStringInfo(&str);
+
 	appendStringInfo(&str, "create table %s.%s as (select ", 
 			quote_identifier(sampleSchemaName), quote_identifier(sampleTableName)); 
 	
@@ -1387,7 +1381,7 @@ static Oid buildSampleTable(Oid relationOid,
 		}
 	}
 	
-	// if table is partitioned, we create a sample over all parts
+	/* if table is partitioned, we create a sample over all parts */
 	appendStringInfo(&str, "from %s.%s as Ta where random() < %.38f limit %lu) distributed
randomly", 
 			quote_identifier(schemaName), 
 			quote_identifier(tableName), randomThreshold, (unsigned long) requestedSampleSize);
@@ -1419,14 +1413,7 @@ static Oid buildSampleTable(Oid relationOid,
 				quote_identifier(tableName));
 	}
 	
-	/* 
-	 * Update the sample table's reltuples, relpages. Without these, the queries to the sample
table would call cdbRelsize which can be an expensive call. 
-	 * We know the number of tuples in the sample table, but don't have the information about
the number of pages. We set it to 2 arbitrarily.
-	 */
-	updateReltuplesRelpagesInCatalog(sampleTableOid, *sampleTableRelTuples, 2);
-
 	pfree((void *) rangeVar);
-	pfree((void *) sampleTableName);
 	pfree((void *) tableName);
 	pfree((void *) schemaName);
 	pfree((void *) sampleSchemaName);
@@ -1436,17 +1423,18 @@ static Oid buildSampleTable(Oid relationOid,
 /**
  * Drops the sample table created during ANALYZE.
  */
-static void dropSampleTable(Oid sampleTableOid)
+void dropSampleTable(Oid sampleTableOid, bool isExternal)
 {
 	StringInfoData str;
 	const char *sampleSchemaName = NULL;
 	const char *sampleTableName = NULL;
 
-	sampleSchemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); //must be pfreed
-	sampleTableName = get_rel_name(sampleTableOid); // must be pfreed 	
+	sampleSchemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); /* must be pfreed
*/
+	sampleTableName = get_rel_name(sampleTableOid); /* must be pfreed */
 
 	initStringInfo(&str);
-	appendStringInfo(&str, "drop table %s.%s", 
+	appendStringInfo(&str, "drop %stable %s.%s",
+			isExternal ? "external " : "",
 			quote_identifier(sampleSchemaName), 
 			quote_identifier(sampleTableName));
 	
@@ -1458,7 +1446,6 @@ static void dropSampleTable(Oid sampleTableOid)
 	pfree((void *)sampleTableName);
 }
 
-
 /**
  * This method determines the number of pages corresponding to an index.
  * Input:
@@ -1766,8 +1753,8 @@ static float4 analyzeComputeNDistinctAbsolute(Oid sampleTableOid,
 	const char *sampleSchemaName = NULL;
 	const char *sampleTableName = NULL;
 	
-	sampleSchemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); //must be pfreed
-	sampleTableName = get_rel_name(sampleTableOid); // must be pfreed 	
+	sampleSchemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); /* must be pfreed
*/
+	sampleTableName = get_rel_name(sampleTableOid); /* must be pfreed */
 
 	initStringInfo(&str);
 	appendStringInfo(&str, "select count(*)::float4 from (select Ta.%s from %s.%s as Ta
group by Ta.%s) as Tb",
@@ -1807,14 +1794,14 @@ static float4 analyzeComputeNRepeating(Oid relationOid,
 	const char *sampleSchemaName = NULL;
 	const char *sampleTableName = NULL;
 	
-	sampleSchemaName = get_namespace_name(get_rel_namespace(relationOid)); //must be pfreed
-	sampleTableName = get_rel_name(relationOid); // must be pfreed 	
+	sampleSchemaName = get_namespace_name(get_rel_namespace(relationOid)); /* must be pfreed
*/
+	sampleTableName = get_rel_name(relationOid); /* must be pfreed */
 
 	initStringInfo(&str);
 	appendStringInfo(&str, "select count(v)::float4 from (select Ta.%s as v, count(Ta.%s)
as f from %s.%s as Ta group by Ta.%s) as foo where f > 1",
-			quote_identifier(attributeName), 
 			quote_identifier(attributeName),
-			quote_identifier(sampleSchemaName), 
+			quote_identifier(attributeName),
+			quote_identifier(sampleSchemaName),
 			quote_identifier(sampleTableName),
 			quote_identifier(attributeName));
 
@@ -1838,7 +1825,7 @@ static float4 analyzeComputeNRepeating(Oid relationOid,
  * Output:
  * 	array of attribute type
  */
-static ArrayType * SPIResultToArray(int resultAttributeNumber, MemoryContext allocationContext)
+ArrayType * SPIResultToArray(int resultAttributeNumber, MemoryContext allocationContext)
 {
 	ArrayType *result = NULL;
 	int i = 0;
@@ -1962,8 +1949,8 @@ static float4 analyzeNullCount(Oid sampleTableOid, Oid relationOid,
const char *
 		const char *schemaName = NULL;
 		const char *tableName = NULL;
 
-		schemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); //must be pfreed
-		tableName = get_rel_name(sampleTableOid); // must be pfreed
+		schemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); /* must be pfreed */
+		tableName = get_rel_name(sampleTableOid); /* must be pfreed */
 
 		initStringInfo(&str);
 		appendStringInfo(&str, "select count(*)::float4 from %s.%s as Ta where Ta.%s is null",
@@ -2064,8 +2051,8 @@ static float4 analyzeComputeAverageWidth(Oid sampleTableOid,
 		const char *sampleSchemaName = NULL;
 		const char *sampleTableName = NULL;
 
-		sampleSchemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); //must be pfreed
-		sampleTableName = get_rel_name(sampleTableOid); // must be pfreed
+		sampleSchemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); /* must be pfreed
*/
+		sampleTableName = get_rel_name(sampleTableOid); /* must be pfreed */
 
 		initStringInfo(&str);
 		appendStringInfo(&str, "select avg(pg_column_size(Ta.%s))::float4 from %s.%s as Ta
where Ta.%s is not null",
@@ -2130,8 +2117,8 @@ static void analyzeComputeMCV(Oid relationOid,
 		Assert(relTuples > 0.0);
 		Assert(nEntries > 0);
 
-		sampleSchemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); //must be pfreed
-		sampleTableName = get_rel_name(sampleTableOid); // must be pfreed
+		sampleSchemaName = get_namespace_name(get_rel_namespace(sampleTableOid)); /* must be pfreed
*/
+		sampleTableName = get_rel_name(sampleTableOid); /* must be pfreed */
 
 		initStringInfo(&str);
 		appendStringInfo(&str, "select Ta.%s as v, count(Ta.%s)::float4/%f::float4 as f from
%s.%s as Ta "
@@ -2880,7 +2867,6 @@ static void updateAttributeStatisticsInCatalog(Oid relationOid, const
char *attr
 
 }
 
-
 /**
  * This method estimates the number of tuples and pages in a heaptable relation. Getting
the number of blocks is straightforward.
  * Estimating the number of tuples is a little trickier. There are two factors that complicate
this:
@@ -2966,7 +2952,7 @@ static void gp_statistics_estimate_reltuples_relpages_heap(Relation
rel, float4
 			 * could get added to it, but we ignore such tuples.
 			 */
 
-			// -------- MirroredLock ----------
+			/* -------- MirroredLock ---------- */
 			MIRROREDLOCK_BUFMGR_LOCK;
 
 			targbuffer = ReadBuffer(rel, targblock);
@@ -3002,7 +2988,7 @@ static void gp_statistics_estimate_reltuples_relpages_heap(Relation
rel, float4
 			UnlockReleaseBuffer(targbuffer);
 
 			MIRROREDLOCK_BUFMGR_UNLOCK;
-			// -------- MirroredLock ----------
+			/* -------- MirroredLock ---------- */
 
 			nblocksseen++;
 		}		
@@ -3107,44 +3093,3 @@ static void gp_statistics_estimate_reltuples_relpages_parquet(Relation
rel, floa
 	pfree(fstotal);
 	return;
 }
-
-/* --------------------------------
- *		gp_statistics_estimate_reltuples_relpages_external_pxf -
- *
- *		Fetch reltuples and relpages for an external table which is PXF
- * --------------------------------
- */
-void gp_statistics_estimate_reltuples_relpages_external_pxf(Relation rel, StringInfo location,
-															float4 *reltuples, float4 *relpages,
-															StringInfo err_msg)
-{
-
-	PxfStatsElem *elem = NULL;
-	elem = get_pxf_statistics(location->data, rel, err_msg);
-
-	/*
-	 * if get_pxf_statistics returned NULL - probably a communication error, we fall back to
former values
-	 * for the relation (can be default if no analyze was run successfully before)
-	 * we don't want to stop the analyze, since this can be part of a long procedure performed
on many tables
-	 * not just this one
-	 */
-	if (!elem)
-	{
-		*relpages = rel->rd_rel->relpages;
-		*reltuples = rel->rd_rel->reltuples;
-		return;
-	}
-	
-	*relpages = floor(( ((float4)elem->blockSize) * elem->numBlocks) / BLCKSZ);
-	*reltuples = elem->numTuples;
-	/* relpages can't be 0 if there are tuples in the table. */
-	if ((*relpages < 1.0) && (*reltuples > 0))
-		*relpages = 1.0;
-	pfree(elem);
-	
-	/* in case there were problems with the PXF service, keep the defaults */
-	if (*relpages < 0)
-		*relpages =  gp_external_table_default_number_of_pages;
-	if (*reltuples < 0)
-		*reltuples =  gp_external_table_default_number_of_tuples;
-}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/backend/commands/tablecmds.c
----------------------------------------------------------------------
diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c
index 547725e..b47bdce 100644
--- a/src/backend/commands/tablecmds.c
+++ b/src/backend/commands/tablecmds.c
@@ -895,11 +895,11 @@ static Datum AddDefaultPageRowGroupSize(Datum relOptions, List *defList){
 * In here we first dispatch a normal DefineRelation() (with relstorage
 * external) in order to create the external relation entries in pg_class
 * pg_type etc. Then once this is done we dispatch ourselves (DefineExternalRelation)
-* in order to create the pg_exttable entry accross the gp array and we
+* in order to create the pg_exttable entry across the gp array and we
 * also record a dependency with the error table, if one exists.
 *
 * Why don't we just do all of this in one dispatch run? because that
-* involves duplicating the DefineRelation() code or severly modifying it
+* involves duplicating the DefineRelation() code or severely modifying it
 * to have special cases for external tables. IMHO it's better and cleaner
 * to leave it intact and do another dispatch.
 * ----------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/backend/optimizer/util/plancat.c
----------------------------------------------------------------------
diff --git a/src/backend/optimizer/util/plancat.c b/src/backend/optimizer/util/plancat.c
index 2258efe..b15b32e 100644
--- a/src/backend/optimizer/util/plancat.c
+++ b/src/backend/optimizer/util/plancat.c
@@ -21,7 +21,6 @@
 #include "access/genam.h"
 #include "access/heapam.h"
 #include "access/parquetsegfiles.h"
-#include "access/pxfuriparser.h"
 #include "catalog/catquery.h"
 #include "catalog/gp_policy.h"
 #include "catalog/pg_inherits.h"
@@ -80,9 +79,6 @@ cdb_estimate_rel_size(RelOptInfo   *relOptInfo,
 static void
 cdb_default_stats_warning_for_index(Oid reloid, Oid indexoid);
 
-static bool 
-need_to_get_stats_pxf(Relation rel, StringInfo location, BlockNumber relpages, double reltuples);
-
 extern BlockNumber RelationGuessNumberOfBlocks(double totalbytes);
 
 /*
@@ -395,20 +391,7 @@ cdb_estimate_rel_size(RelOptInfo   *relOptInfo,
 	 * Asking the QE for the size of the relation is a bit expensive.
 	 * Do we want to do it all the time?  Or only for tables that have never had analyze run?
 	 */
-	if (need_to_get_stats_pxf(rel, &location, relpages, reltuples))
-	{
-		/*
-		 * rel is a pxf external table, and it wasn't yet ANALYZE'ed.
-		 */
-
-		float4 tuples, pages;
-		gp_statistics_estimate_reltuples_relpages_external_pxf(rel, &location, &tuples,
&pages, NULL);
-		
-		relpages = curpages = pages;
-		reltuples = tuples;
-		pfree(location.data);
-	}	
-	else if (relpages > 0) 
+	if (relpages > 0)
 	{
 
 		/*
@@ -420,7 +403,7 @@ cdb_estimate_rel_size(RelOptInfo   *relOptInfo,
 
 		curpages = relpages;
 	}
-	else /* relpages is 0 and this is a regular table or an external non-PXF table */
+	else /* relpages is 0 and this is a regular table or an external table */
 	{
 
 		/*
@@ -500,24 +483,6 @@ cdb_estimate_rel_size(RelOptInfo   *relOptInfo,
 
 }                               /* cdb_estimate_rel_size */
 
-/* 
- * need_to_get_stats_pxf
- *
- * 1. Table is PXF external table, and
- * 2. ANALYZE was not run on the table, and
- * 3. GUC pxf_enable_stat_collection is on
- */
-static bool need_to_get_stats_pxf(Relation rel,
-								 StringInfo loc, 	
-								 BlockNumber relpages,
-								 double		reltuples)
-{
-	return pxf_enable_stat_collection &&
-		   RelationIsExternalPxfReadOnly(rel, loc) &&
-		   relpages == gp_external_table_default_number_of_pages &&
-		   reltuples == gp_external_table_default_number_of_tuples;
-}
-
 /*
  * estimate_rel_size - estimate # pages and # tuples in a table or index
  *

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 60c48c4..7e7d1b8 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -614,6 +614,7 @@ show_allow_system_table_mods(void);
 /* Extension Framework GUCs */
 bool   pxf_enable_filter_pushdown = true;
 bool   pxf_enable_stat_collection = true;
+int    pxf_stat_max_fragments = 100;
 bool   pxf_enable_locality_optimizations = true;
 bool   pxf_isilon = false; /* temporary GUC */
 int    pxf_service_port = 51200; /* temporary GUC */
@@ -6264,6 +6265,16 @@ static struct config_int ConfigureNamesInt[] =
 	},
 
 	{
+		{"pxf_stat_max_fragments", PGC_USERSET, EXTERNAL_TABLES,
+			gettext_noop("Max number of fragments to be sampled during ANALYZE on a PXF table."),
+			NULL,
+			GUC_GPDB_ADDOPT
+		},
+		&pxf_stat_max_fragments,
+		100, 1, INT_MAX, NULL, NULL
+	},
+
+	{
 		{"pxf_service_port", PGC_POSTMASTER, EXTERNAL_TABLES,
 			gettext_noop("PXF service port"),
 			NULL,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/include/access/hd_work_mgr.h
----------------------------------------------------------------------
diff --git a/src/include/access/hd_work_mgr.h b/src/include/access/hd_work_mgr.h
index ea5c8d6..9280df5 100644
--- a/src/include/access/hd_work_mgr.h
+++ b/src/include/access/hd_work_mgr.h
@@ -20,15 +20,15 @@ extern char** map_hddata_2gp_segments(char *uri, int total_segs, int working_seg
 extern void free_hddata_2gp_segments(char **segs_work_map, int total_segs);
 
 /*
- * Structure that describes one Statistics element received from the PXF service
+ * Structure that describes fragments statistics element received from PXF service
  */
-typedef struct sPxfStatsElem
+typedef struct sPxfFragmentStatsElem
 {
-	int   blockSize; /* size of a block size in the PXF target datasource */
-	int   numBlocks;
-	int   numTuples;
-} PxfStatsElem;
-PxfStatsElem *get_pxf_statistics(char *uri, Relation rel, StringInfo err_msg);
+	int numFrags;
+	float4 firstFragSize; /* size of the first fragment */
+	float4 totalSize; /* size of the total datasource */
+} PxfFragmentStatsElem;
+PxfFragmentStatsElem *get_pxf_fragments_statistics(char *uri, Relation rel);
 
 List *get_pxf_hcat_metadata(char *relation_location);
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/include/access/pxfanalyze.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxfanalyze.h b/src/include/access/pxfanalyze.h
new file mode 100644
index 0000000..d641c66
--- /dev/null
+++ b/src/include/access/pxfanalyze.h
@@ -0,0 +1,37 @@
+/*-------------------------------------------------------------------------
+*
+* pxfanalyze.h
+*	  Helper functions to perform ANALYZE on PXF tables.
+*
+* Copyright (c) 2007-2008, Greenplum inc
+*
+*-------------------------------------------------------------------------
+*/
+#ifndef PXFANALYZE_H
+#define PXFANALYZE_H
+
+#include "c.h"
+#include "utils/rel.h"
+#include "nodes/pg_list.h"
+#include "lib/stringinfo.h"
+
+/*
+ * Creates a sample table with data from a PXF table.
+ */
+extern Oid buildPxfSampleTable(Oid relationOid,
+		char* sampleTableName,
+		List *lAttributeNames,
+		float4	relTuples,
+		float4  relFrags,
+		float4 	requestedSampleSize,
+		float4 *sampleTableRelTuples);
+/*
+ * get tuple count estimate, page count estimate (which is
+ * the number of fragments) of a given PXF table.
+ */
+void analyzePxfEstimateReltuplesRelpages(Relation relation,
+		StringInfo location,
+		float4* estimatedRelTuples,
+		float4* estimatedRelPages);
+
+#endif   /* PXFANALYZE_H */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/include/access/pxfmasterapi.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxfmasterapi.h b/src/include/access/pxfmasterapi.h
index 371d743..020351c 100644
--- a/src/include/access/pxfmasterapi.h
+++ b/src/include/access/pxfmasterapi.h
@@ -55,7 +55,7 @@ typedef struct sFragmentHost
 extern List* get_datanode_rest_servers(GPHDUri *hadoop_uri, ClientContext* client_context);
 extern void free_datanode_rest_servers(List *srvrs);
 extern void free_datanode_rest_server(PxfServer* srv);
-extern PxfStatsElem *get_data_statistics(GPHDUri* hadoop_uri, ClientContext *cl_context,
StringInfo err_msg);
+extern PxfFragmentStatsElem *get_fragments_statistics(GPHDUri* hadoop_uri, ClientContext
*cl_context);
 extern List* get_data_fragment_list(GPHDUri *hadoop_uri,  ClientContext* client_context);
 extern void free_fragment(DataFragment *fragment);
 extern List* get_hcat_metadata(GPHDUri* hadoop_uri, char *location, ClientContext *client_context);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/include/access/pxfuriparser.h
----------------------------------------------------------------------
diff --git a/src/include/access/pxfuriparser.h b/src/include/access/pxfuriparser.h
index 334deb3..4e239a5 100644
--- a/src/include/access/pxfuriparser.h
+++ b/src/include/access/pxfuriparser.h
@@ -12,7 +12,7 @@
  * All PXF's resources are under /PXF_SERVICE_PREFIX/PXF_VERSION/...
  */
 #define PXF_SERVICE_PREFIX "pxf"
-#define PXF_VERSION "v13" /* PXF version */
+#define PXF_VERSION "v14" /* PXF version */
 
 /*
  * FragmentData - describes a single Hadoop file split / HBase table region

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/include/cdb/cdbanalyze.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbanalyze.h b/src/include/cdb/cdbanalyze.h
index 7cc2f01..1840c7d 100644
--- a/src/include/cdb/cdbanalyze.h
+++ b/src/include/cdb/cdbanalyze.h
@@ -34,10 +34,4 @@
 extern const int gp_external_table_default_number_of_pages;
 extern const int gp_external_table_default_number_of_tuples;
 
-void gp_statistics_estimate_reltuples_relpages_external_pxf(Relation rel,
-															StringInfo location,
-															float4 *reltuples,
-															float4 *relpages,
-															StringInfo err_msg);
-
 #endif   /* CDBANALYZE_H */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/include/commands/analyzeutils.h
----------------------------------------------------------------------
diff --git a/src/include/commands/analyzeutils.h b/src/include/commands/analyzeutils.h
index 0ac3d20..59706a3 100644
--- a/src/include/commands/analyzeutils.h
+++ b/src/include/commands/analyzeutils.h
@@ -12,6 +12,8 @@
 #ifndef ANALYZEUTILS_H
 #define ANALYZEUTILS_H
 
+#include "utils/array.h"
+
 /* extern functions called by commands/analyze.c */
 extern void aggregate_leaf_partition_MCVs(Oid relationOid,
 		AttrNumber attnum,
@@ -23,4 +25,37 @@ extern void aggregate_leaf_partition_histograms(Oid relationOid,
 		ArrayType **result);
 extern bool datumCompare(Datum d1, Datum d2, Oid opFuncOid);
 
+/*
+ * Helper functions for ANALYZE.
+ */
+
+/**
+ * Drops the sample table created during ANALYZE.
+ */
+extern void dropSampleTable(Oid sampleTableOid, bool isExternal);
+
+/**
+ * Generates a table name for the auxiliary sample table that may be created during ANALYZE.
+ */
+extern char* temporarySampleTableName(Oid relationOid, char* prefix);
+
+/* Convenience */
+extern ArrayType * SPIResultToArray(int resultAttributeNumber, MemoryContext allocationContext);
+
+/* spi execution helpers */
+typedef void (*spiCallback)(void *clientDataOut);
+extern void spiExecuteWithCallback(const char *src, bool read_only, long tcount,
+           spiCallback callbackFn, void *clientData);
+
+typedef struct
+{
+    int numColumns;
+    MemoryContext memoryContext;
+    ArrayType ** output;
+} EachResultColumnAsArraySpec;
+
+extern void spiCallback_getEachResultColumnAsArray(void *clientData);
+extern void spiCallback_getProcessedAsFloat4(void *clientData);
+extern void spiCallback_getSingleResultRowColumnAsFloat4(void *clientData);
+
 #endif  /* ANALYZEUTILS_H */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/81385f09/src/include/utils/guc.h
----------------------------------------------------------------------
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index 14ed6a2..1623ce8 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -454,6 +454,7 @@ extern char   *gp_snmp_debug_log;
 /* Extension Framework GUCs */
 extern bool   pxf_enable_filter_pushdown; /* turn pushdown logic on/off     */
 extern bool   pxf_enable_stat_collection; /* turn off stats collection if needed */
+extern int    pxf_stat_max_fragments; /* max fragments to be sampled during analyze */
 extern bool   pxf_enable_locality_optimizations; /* turn locality optimization in the data
allocation algorithm on/off     */
 /*
  * Is Isilon the target storage system ?



Mime
View raw message