hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h...@apache.org
Subject incubator-hawq git commit: HAWQ-808. Refactor feature test for external_oid with new framework.
Date Tue, 12 Jul 2016 06:50:41 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 651ce19d1 -> bab04b57b


HAWQ-808. Refactor feature test for external_oid with new framework.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bab04b57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bab04b57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bab04b57

Branch: refs/heads/master
Commit: bab04b57bfc4737b584595e79f61a7afec523c05
Parents: 651ce19
Author: xunzhang <xunzhangthu@gmail.com>
Authored: Tue Jul 12 14:14:08 2016 +0800
Committer: xunzhang <xunzhangthu@gmail.com>
Committed: Tue Jul 12 14:23:34 2016 +0800

----------------------------------------------------------------------
 .../ExternalSource/ans/external_oid.ans.source  | 209 ++++++
 .../ExternalSource/data/multi_table.json        |   1 +
 .../ExternalSource/data/single_table.json       |   1 +
 src/test/feature/ExternalSource/lib/Makefile    |  44 ++
 src/test/feature/ExternalSource/lib/function.c  | 727 +++++++++++++++++++
 .../ExternalSource/sql/external_oid.sql.source  | 108 +++
 .../ExternalSource/test_external_oid.cpp        |  35 +
 src/test/feature/Makefile                       |   2 +
 src/test/regress/input/external_oid.source      | 118 ---
 src/test/regress/known_good_schedule            |   1 -
 src/test/regress/output/external_oid.source     | 211 ------
 11 files changed, 1127 insertions(+), 330 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/ans/external_oid.ans.source
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/ans/external_oid.ans.source b/src/test/feature/ExternalSource/ans/external_oid.ans.source
new file mode 100644
index 0000000..3f3f3ae
--- /dev/null
+++ b/src/test/feature/ExternalSource/ans/external_oid.ans.source
@@ -0,0 +1,209 @@
+-- --------------------------------------
+-- test first external Oid initialization
+-- --------------------------------------
+-- start_matchsubs
+--
+-- # create a match/subs expression to handle ip addresses that change
+--
+-- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/
+-- s/oid \d+/oid SOME_OID/
+--
+-- m/.*deleted tuple oid=\d+ from heap table pg_class.*/
+-- s/oid=\d+/oid=OID/
+--
+-- end_matchsubs
+-- Create function that returns the first external Oid boundary
+CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid
+  AS '@SHARE_LIBRARY_PATH@', 'min_external_oid'
+  LANGUAGE C;
+CREATE FUNCTION
+-- Create function that returns the current external Oid
+CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid
+  AS '@SHARE_LIBRARY_PATH@', 'get_next_external_oid'
+  LANGUAGE C;
+CREATE FUNCTION
+-- Create function that sets the current external Oid
+CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid
+  AS '@SHARE_LIBRARY_PATH@', 'set_next_external_oid'
+  LANGUAGE C;
+CREATE FUNCTION
+-- Create function to insert and scan in-memory data to pg_class
+CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text
+  AS '@SHARE_LIBRARY_PATH@', 'load_json_data'
+  LANGUAGE C;
+CREATE FUNCTION
+-- Create function that inserts tuple with given Oid
+CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text
+  AS '@SHARE_LIBRARY_PATH@', 'caql_insert_into_heap_pg_class'
+  LANGUAGE C;
+CREATE FUNCTION
+-- Create function that inserts tuple with given Oid
+CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text
+  AS '@SHARE_LIBRARY_PATH@', 'caql_delete_from_heap_pg_class'
+  LANGUAGE C;
+CREATE FUNCTION
+-- --------------------------------------
+-- Test hcat table external oid initialization
+-- --------------------------------------
+-- Boundary should be at FirstExternalObjectId
+--SELECT min_external_oid();
+-- NextExternalObjectId is uninitialized
+SELECT get_next_external_oid();
+ get_next_external_oid
+-----------------------
+                     0
+(1 row)
+
+SELECT load_json_data('@abs_datadir@/single_table.json');
+  load_json_data
+------------------
+ default.mytable
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+        0
+(1 row)
+
+BEGIN TRANSACTION;
+BEGIN
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+        0
+(1 row)
+
+-- load default.mytable -> +3 oids
+-- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype
+SELECT load_json_data('@abs_datadir@/single_table.json');
+  load_json_data
+------------------
+ default.mytable
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column? 
+----------
+        3
+(1 row)
+
+-- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids
+-- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2)
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+      load_json_data
+--------------------------
+ db1.ht1 db2.ht1 db2.ht2
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+       11
+(1 row)
+
+END TRANSACTION;
+COMMIT
+-- New transaction will reset external Oid start point
+-- Yields the same result as previous transaction
+BEGIN TRANSACTION;
+BEGIN
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+        0
+(1 row)
+
+SELECT load_json_data('@abs_datadir@/single_table.json');
+  load_json_data
+------------------
+ default.mytable
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+        3
+(1 row)
+
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+      load_json_data
+--------------------------
+ db1.ht1 db2.ht1 db2.ht2
+(1 row)
+
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+       11
+(1 row)
+
+END TRANSACTION;
+COMMIT
+-- --------------------------------------
+-- Test external oid rollover
+-- --------------------------------------
+BEGIN TRANSACTION;
+BEGIN
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+ ?column?
+----------
+        0
+(1 row)
+
+SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0;
+ ?column?
+----------
+ t
+(1 row)
+
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+      load_json_data
+--------------------------
+ db1.ht1 db2.ht1 db2.ht2
+(1 row)
+
+-- Used up external Oids result in Oid overflow
+SELECT get_next_external_oid();
+ get_next_external_oid
+-----------------------
+                     0
+(1 row)
+
+-- Rollover disallowed!
+SELECT load_json_data('@abs_datadir@/single_table.json');
+psql:/tmp/TestExternalOid_TestExternalOidAll.sql:92: ERROR:  number of external objects from HCatalog exceeded 10M during transaction
+HINT:  Separate HCatalog queries into different transactions to process.
+END TRANSACTION;
+ROLLBACK
+-- --------------------------------------
+-- Test external Oid boundary
+-- --------------------------------------
+-- Create a tuple with Oid larger than FirstExternalObjectId
+-- Will fail during next session when try to query HCatalog
+-- Because external Oid boundary is violated
+SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl');
+                      caql_insert_into_heap_pg_class
+--------------------------------------------------------------------------
+ inserted tuple to heap table pg_class (oid 4284481555, relname table_xl)
+(1 row)
+
+-- cleanup
+SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20);
+            caql_delete_from_heap_pg_class
+-------------------------------------------------------
+ deleted tuple oid=4284481555 from heap table pg_class
+(1 row)
+
+DROP FUNCTION caql_delete_from_heap_pg_class(relid oid);
+DROP FUNCTION
+DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text);
+DROP FUNCTION
+DROP FUNCTION load_json_data(filename text);
+DROP FUNCTION
+DROP FUNCTION get_next_external_oid();
+DROP FUNCTION
+DROP FUNCTION set_next_external_oid(ext_oid oid);
+DROP FUNCTION
+DROP FUNCTION min_external_oid();
+DROP FUNCTION

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/data/multi_table.json
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/data/multi_table.json b/src/test/feature/ExternalSource/data/multi_table.json
new file mode 100644
index 0000000..82e70b0
--- /dev/null
+++ b/src/test/feature/ExternalSource/data/multi_table.json
@@ -0,0 +1 @@
+{"PXFMetadata":[{"item":{"path":"db1","name":"ht1"},"fields":[{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"vc2","type":"varchar","modifiers":["3"],"sourceType":"varchar"}]},{"item":{"path":"db2","name":"ht1"},"fields":[{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"vc2","type":"varchar","modifiers":["3"],"sourceType":"varchar"}]},{"item":{"path":"db2","name":"ht2"},"fields":[{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"vc2","type":"varchar","modifiers":["3"],"sourceType":"varchar"}]}]}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/data/single_table.json
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/data/single_table.json b/src/test/feature/ExternalSource/data/single_table.json
new file mode 100644
index 0000000..b571e5d
--- /dev/null
+++ b/src/test/feature/ExternalSource/data/single_table.json
@@ -0,0 +1 @@
+{"PXFMetadata":[{"item":{"path":"default","name":"mytable"},"fields":[{"name":"s1","type":"text","sourceType":"string"},{"name":"s2","type":"text","sourceType":"string"},{"name":"n1","type":"int4","sourceType":"int"},{"name":"d1","type":"float8","sourceType":"double"},{"name":"dc1","type":"numeric","modifiers":["38","18"],"sourceType":"decimal"},{"name":"tm","type":"timestamp","sourceType":"timestamp"},{"name":"f","type":"float4","sourceType":"float"},{"name":"bg","type":"int8","sourceType":"bigint"},{"name":"b","type":"bool","sourceType":"boolean"},{"name":"tn","type":"int2","sourceType":"tinyint"},{"name":"sml","type":"int2","sourceType":"tinyint"},{"name":"dt","type":"date","sourceType":"date"},{"name":"vc1","type":"varchar","modifiers":["5"],"sourceType":"varchar"},{"name":"c1","type":"bpchar","modifiers":["3"],"sourceType":"char"},{"name":"bin","type":"bytea","sourceType":"binary"}]}]}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/lib/Makefile
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/lib/Makefile b/src/test/feature/ExternalSource/lib/Makefile
new file mode 100644
index 0000000..46f6c20
--- /dev/null
+++ b/src/test/feature/ExternalSource/lib/Makefile
@@ -0,0 +1,44 @@
+top_builddir = ../../../../..
+include $(top_builddir)/src/Makefile.global
+
+OS = $(shell uname)
+
+CXX = gcc 
+CXXFLAGS = -Wall -O1 -g -std=gnu99 -Wmissing-prototypes -Wpointer-arith -Wendif-labels -Wformat-security -fno-strict-aliasing -fwrapv -fPIC
+
+CC = gcc
+CFLAGS = -Wall -O1 -g -std=gnu99 -Wmissing-prototypes -Wpointer-arith -Wendif-labels -Wformat-security -fno-strict-aliasing -fwrapv -fPIC
+
+CPPFLAGS = -I$(abs_top_srcdir)/src/include
+CPPFLAGS += -I$(abs_top_srcdir)/depends/libhdfs3/build/install$(prefix)/include
+CPPFLAGS += -I$(abs_top_srcdir)/depends/libyarn/build/install$(prefix)/include
+
+LDFLAGS = -L$(libdir)
+LDFLAGS += -L$(abs_top_srcdir)/src/port
+LDFLAGS += -L$(abs_top_builddir)/src/port
+LDFLAGS += -L$(abs_top_srcdir)/depends/libhdfs3/build/install$(prefix)/lib
+LDFLAGS += -L$(abs_top_srcdir)/depends/libyarn/build/install$(prefix)/lib
+
+POSTGRES = $(abs_top_srcdir)/src/backend/postgres
+
+PROG = function.c
+OBJS = function.o
+TARGET = function.so
+
+RM = rm -rf 
+
+all: $(TARGET)
+
+$(OBJS): $(PROG)
+	$(CXX) $(CXXFLAGS) $(CPPFLAGS) -c -o $(OBJS) $(PROG)
+
+$(TARGET): $(OBJS)
+ifeq ($(OS),Darwin)
+	$(CXX) $(CXXFLAGS) -bundle $(OBJS) -bundle_loader $(POSTGRES) $(LDFLAGS) -o $@
+else
+	$(CXX) $(CXXFLAGS) -shared $(OBJS) $(LDFLAGS) -Wl,-rpath,'$(abs_top_builddir)/lib' -o $@
+endif
+
+clean:
+	$(RM) *.o
+	$(RM) *.so

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/lib/function.c
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/lib/function.c b/src/test/feature/ExternalSource/lib/function.c
new file mode 100644
index 0000000..4275edc
--- /dev/null
+++ b/src/test/feature/ExternalSource/lib/function.c
@@ -0,0 +1,727 @@
+/*
+ * json_utils.c
+ *
+ *  Created on: Mar 5, 2015
+ *      Author: antova
+ */
+
+#include "catalog/external/externalmd.h"
+#include "postgres.h"
+#include "access/hd_work_mgr.h"
+#include "funcapi.h"
+#include "catalog/catquery.h"
+#include "catalog/gp_policy.h"
+#include "catalog/namespace.h"
+#include "catalog/pg_database.h"
+#include "catalog/pg_exttable.h"
+#include "catalog/pg_namespace.h"
+#include "cdb/cdbinmemheapam.h"
+#include "nodes/makefuncs.h"
+#include "utils/array.h"
+#include "utils/builtins.h"
+#include "utils/lsyscache.h"
+#include "access/transam.h"
+
+/*
+ * number of output columns for the UDFs for scanning in memory catalog tables
+ */
+#define NUM_COLS 3
+
+#ifdef PG_MODULE_MAGIC
+PG_MODULE_MAGIC;
+#endif
+
+static 
+char *read_file(const char *filename);
+
+static
+Oid GetRelationOid(char *tableName);
+
+/* utility functions for loading json files */
+extern Datum load_json_data(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_pg_namespace(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_pg_type(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_gp_distribution_policy(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS);
+extern Datum caql_scan_in_memory_pg_attribute(PG_FUNCTION_ARGS);
+extern Datum caql_insert_into_heap_pg_class(PG_FUNCTION_ARGS);
+extern Datum caql_delete_from_heap_pg_class(PG_FUNCTION_ARGS);
+extern Datum get_next_external_oid(PG_FUNCTION_ARGS);
+extern Datum set_next_external_oid(PG_FUNCTION_ARGS);
+extern Datum min_external_oid(PG_FUNCTION_ARGS);
+
+PG_FUNCTION_INFO_V1(load_json_data);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_namespace);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_type);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_gp_distribution_policy);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_exttable);
+PG_FUNCTION_INFO_V1(caql_scan_in_memory_pg_attribute);
+PG_FUNCTION_INFO_V1(caql_insert_into_heap_pg_class);
+PG_FUNCTION_INFO_V1(caql_delete_from_heap_pg_class);
+PG_FUNCTION_INFO_V1(get_next_external_oid);
+PG_FUNCTION_INFO_V1(set_next_external_oid);
+PG_FUNCTION_INFO_V1(min_external_oid);
+
+Datum
+load_json_data(PG_FUNCTION_ARGS)
+{
+	text *filename = PG_GETARG_TEXT_P(0);
+	char *filenameStr = text_to_cstring(filename);
+
+	char *pcbuf = read_file(filenameStr);
+	StringInfoData buf;
+	initStringInfo(&buf);
+	appendStringInfo(&buf, "%s", pcbuf);
+    
+	List *items = ParsePxfEntries(&buf, HiveProfileName, HcatalogDbOid);
+	pfree(buf.data);
+	
+	StringInfoData tblNames;
+	initStringInfo(&tblNames);
+	ListCell *lc = NULL;
+	foreach (lc, items)
+	{
+		PxfItem *item = (PxfItem *) lfirst(lc);
+		appendStringInfo(&tblNames, "%s.%s ", item->path, item->name);
+	}
+	
+	PG_RETURN_TEXT_P(cstring_to_text(tblNames.data));
+}
+
+char *
+read_file(const char *filename)
+{
+	FILE *pf = fopen(filename, "r");
+	if (NULL == pf)
+	{
+		elog(ERROR, "Could not open file %s for reading", filename);
+	}
+	
+	StringInfoData strinfo;
+	initStringInfo(&strinfo);
+
+	const int len = 1024;
+	char buf[len];
+	
+	while (NULL != fgets(buf, len, pf))
+	{
+		appendStringInfo(&strinfo, "%s", buf);
+	}
+
+	fclose(pf);
+	return strinfo.data;
+}
+
+Datum
+caql_scan_in_memory_pg_namespace(PG_FUNCTION_ARGS)
+{
+	text *namespaceNameText = PG_GETARG_TEXT_P(0);
+	char *namespaceNameStr =  text_to_cstring(namespaceNameText);
+
+	FuncCallContext    *funcctx;
+	Datum				result;
+	MemoryContext		oldcontext;
+	TupleDesc	tupdesc;
+	HeapTuple	restuple;
+	HeapTuple readtup = NULL;
+	cqContext  *pcqCtx;
+	StringInfoData buf;
+	initStringInfo(&buf);
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		funcctx = SRF_FIRSTCALL_INIT();
+		/* switch context when allocating stuff to be used in later calls */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "nspname",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "nspdboid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "oid",
+						   OIDOID, -1, 0);
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/* create tuples for pg_namespace table */
+		pcqCtx = caql_beginscan(
+				NULL,
+				cql("SELECT * FROM pg_namespace "
+						" WHERE nspname = :1 and nspdboid = :2",
+						CStringGetDatum((char *) namespaceNameStr), ObjectIdGetDatum(HcatalogDbOid)));
+
+		funcctx->user_fctx = pcqCtx;
+
+		/* return to original context when allocating transient memory */
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	pcqCtx = (cqContext *)funcctx->user_fctx;
+
+	if (NULL != (readtup = caql_getnext(pcqCtx)))
+	{
+		Datum values[NUM_COLS];
+		bool nulls[NUM_COLS];
+
+		Relation relation = RelationIdGetRelation(NamespaceRelationId);
+		TupleDesc tupleDesc = RelationGetDescr(relation);
+		
+		char *nspname = DatumGetCString(tuple_getattr(readtup, tupleDesc, Anum_pg_namespace_nspname));
+		text *t = cstring_to_text(nspname);
+
+		values[0] = PointerGetDatum(t);
+		nulls[0]  = false;
+
+		values[1] = tuple_getattr(readtup, tupleDesc, Anum_pg_namespace_nspdboid);
+		nulls[1]  = false;
+
+		values[2] = ObjectIdGetDatum(HeapTupleGetOid(readtup));
+		nulls[2]  = false;
+		
+		elog(DEBUG1, "Namespace oid: %d", HeapTupleGetOid(readtup));
+		
+		/* build tuple */
+		restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+		/* make the tuple into a datum */
+		result = HeapTupleGetDatum(restuple);
+
+		RelationClose(relation);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+	else
+	{
+		caql_endscan(pcqCtx);
+		SRF_RETURN_DONE(funcctx);
+	}
+}
+
+Datum
+caql_scan_in_memory_pg_type(PG_FUNCTION_ARGS)
+{
+	text *relNameText = PG_GETARG_TEXT_P(0);
+	char *relNameStr =  text_to_cstring(relNameText);
+
+	FuncCallContext    *funcctx;
+	Datum				result;
+	MemoryContext		oldcontext;
+	TupleDesc	tupdesc;
+	HeapTuple	restuple;
+	HeapTuple readtup = NULL;
+	cqContext  *pcqCtx;
+	StringInfoData buf;
+	initStringInfo(&buf);
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		funcctx = SRF_FIRSTCALL_INIT();
+		/* switch context when allocating stuff to be used in later calls */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "typoid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "typname",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "typnamespace",
+						   OIDOID, -1, 0);
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/* create tuples for pg_type table */
+		pcqCtx = caql_beginscan(
+				NULL,
+				cql("SELECT * FROM pg_type "
+						" WHERE typname = :1",
+						CStringGetDatum((char *) relNameStr)));
+
+		funcctx->user_fctx = pcqCtx;
+
+		/* return to original context when allocating transient memory */
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	pcqCtx = (cqContext *)funcctx->user_fctx;
+
+	if (NULL != (readtup = caql_getnext(pcqCtx)))
+	{
+		Datum values[NUM_COLS];
+		bool nulls[NUM_COLS];
+
+		Relation relation = RelationIdGetRelation(TypeRelationId);
+		TupleDesc tupleDesc = RelationGetDescr(relation);
+
+		values[0] = ObjectIdGetDatum(HeapTupleGetOid(readtup));
+		nulls[0]  = false;
+
+		elog(DEBUG1, "Type oid: %d", HeapTupleGetOid(readtup));
+
+		char *typname = DatumGetCString(tuple_getattr(readtup, tupleDesc, Anum_pg_type_typname));
+		text *t = cstring_to_text(typname);
+
+		values[1] = PointerGetDatum(t);
+		nulls[1]  = false;
+
+		values[2] = tuple_getattr(readtup, tupleDesc, Anum_pg_type_typnamespace);
+		nulls[2]  = false;
+
+		/* build tuple */
+		restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+		/* make the tuple into a datum */
+		result = HeapTupleGetDatum(restuple);
+
+		RelationClose(relation);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+	else
+	{
+		caql_endscan(pcqCtx);
+		SRF_RETURN_DONE(funcctx);
+	}
+}
+
+Datum
+caql_scan_in_memory_gp_distribution_policy(PG_FUNCTION_ARGS)
+{
+	text *tableName = PG_GETARG_TEXT_P(0);
+	char *tableNameStr = text_to_cstring(tableName);
+	Oid reloid = InvalidOid;
+
+	FuncCallContext    *funcctx;
+	Datum				result;
+	MemoryContext		oldcontext;
+	TupleDesc	tupdesc;
+	HeapTuple	restuple;
+	HeapTuple pgclasstup = NULL;
+	HeapTuple readtup = NULL;
+	cqContext  *pcqCtx;
+	StringInfoData buf;
+	initStringInfo(&buf);
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		funcctx = SRF_FIRSTCALL_INIT();
+		/* switch context when allocating stuff to be used in later calls */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "tableoid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "tablename",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "distpolicy",
+						   TEXTOID, -1, 0);
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/* iterate on pg_class and query
+		 * gp_distribution_policy by given oid */
+		pcqCtx = caql_beginscan(
+						NULL,
+						cql("SELECT oid FROM pg_class "
+						" WHERE relname = :1",
+						CStringGetDatum(tableNameStr)));
+
+		funcctx->user_fctx = pcqCtx;
+
+		/* return to original context when allocating transient memory */
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	pcqCtx = (cqContext *)funcctx->user_fctx;
+
+	if (NULL != (pgclasstup = caql_getnext(pcqCtx)))
+	{
+		Datum values[NUM_COLS];
+		bool nulls[NUM_COLS];
+
+		/* create tuples for pg_exttable table */
+		cqContext* pcqCtx1 = caql_beginscan(
+				NULL,
+				cql("SELECT * FROM gp_distribution_policy "
+				" WHERE localoid = :1",
+				ObjectIdGetDatum(HeapTupleGetOid(pgclasstup))));
+
+		Relation relation = RelationIdGetRelation(GpPolicyRelationId);
+		TupleDesc tupleDesc = RelationGetDescr(relation);
+
+		readtup = caql_getnext(pcqCtx1);
+
+		values[0] = ObjectIdGetDatum(reloid);
+		nulls[0]  = false;
+
+		text *t1 = cstring_to_text(tableNameStr);
+
+		values[1] = PointerGetDatum(t1);
+		nulls[1]  = false;
+
+		text *t2 = NULL;
+		bool isnull;
+		Datum policy = heap_getattr(readtup, Anum_gp_policy_attrnums, tupleDesc, &isnull);
+		if (isnull)
+		{
+			t2 = cstring_to_text("null");
+		}
+		else
+		{
+			/* not tested! */
+			Datum* elems = NULL;
+			int nelems;
+			deconstruct_array(DatumGetArrayTypeP(policy),
+					INT2OID, -1, false, 'i',
+					&elems, NULL, &nelems);
+			Assert(nelems > 0);
+			StringInfoData elems_str;
+			initStringInfo(&elems_str);
+			for (int i = 0; i < nelems; i++)
+			{
+				appendStringInfo(&elems_str, "%d", DatumGetInt16(elems[0]));
+				if (i != nelems)
+				{
+					appendStringInfo(&elems_str, ", ");
+				}
+			}
+			t2 = cstring_to_text(elems_str.data);
+			pfree(elems_str.data);
+		}
+
+		values[2] = PointerGetDatum(t2);
+		nulls[2]  = false;
+
+		/* build tuple */
+		restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+		/* make the tuple into a datum */
+		result = HeapTupleGetDatum(restuple);
+
+		RelationClose(relation);
+		/* there should be only one value per oid */
+		Assert(NULL == caql_getnext(pcqCtx1)) ;
+		caql_endscan(pcqCtx1);
+
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+	else
+	{
+		caql_endscan(pcqCtx);
+		SRF_RETURN_DONE(funcctx);
+	}
+}
+
+Datum
+caql_scan_in_memory_pg_exttable(PG_FUNCTION_ARGS)
+{
+	text *tableName = PG_GETARG_TEXT_P(0);
+	char *tableNameStr = text_to_cstring(tableName);
+	Oid reloid = InvalidOid;
+
+	FuncCallContext    *funcctx;
+	Datum				result;
+	MemoryContext		oldcontext;
+	TupleDesc	tupdesc;
+	HeapTuple	restuple;
+	HeapTuple pgclasstup = NULL;
+	HeapTuple readtup = NULL;
+	cqContext  *pcqCtx;
+	StringInfoData buf;
+	initStringInfo(&buf);
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		funcctx = SRF_FIRSTCALL_INIT();
+		/* switch context when allocating stuff to be used in later calls */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "exttableoid",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "exttablename",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "exttablelocation",
+						   TEXTOID, -1, 0);
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/* iterate on pg_class and query
+		 * pg_exttable by given oid */
+		pcqCtx = caql_beginscan(
+				NULL,
+				cql("SELECT oid FROM pg_class "
+				" WHERE relname = :1",
+				CStringGetDatum(tableNameStr)));
+
+		funcctx->user_fctx = pcqCtx;
+
+		/* return to original context when allocating transient memory */
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	pcqCtx = (cqContext *)funcctx->user_fctx;
+
+	if (NULL != (pgclasstup = caql_getnext(pcqCtx)))
+	{
+		Datum values[NUM_COLS];
+		bool nulls[NUM_COLS];
+
+		/* create tuples for pg_exttable table */
+		cqContext* pcqCtx1 = caql_beginscan(
+				NULL,
+				cql("SELECT * FROM pg_exttable "
+				" WHERE reloid = :1",
+				ObjectIdGetDatum(HeapTupleGetOid(pgclasstup))));
+
+		Relation relation = RelationIdGetRelation(ExtTableRelationId);
+		TupleDesc tupleDesc = RelationGetDescr(relation);
+
+        readtup = caql_getnext(pcqCtx1);
+
+		values[0] = ObjectIdGetDatum(reloid);
+		nulls[0]  = false;
+
+		text *t1 = cstring_to_text(tableNameStr);
+
+		values[1] = PointerGetDatum(t1);
+		nulls[1]  = false;
+
+		Datum locations = tuple_getattr(readtup, tupleDesc, Anum_pg_exttable_location);
+		Datum* elems = NULL;
+		int nelems;
+		deconstruct_array(DatumGetArrayTypeP(locations),
+						  TEXTOID, -1, false, 'i',
+						  &elems, NULL, &nelems);
+		Assert(nelems > 0);
+		char *loc_str = DatumGetCString(DirectFunctionCall1(textout, elems[0]));
+		text *t2 = cstring_to_text(loc_str);
+
+		values[2] = PointerGetDatum(t2);
+		nulls[2]  = false;
+
+		/* build tuple */
+		restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+		/* make the tuple into a datum */
+		result = HeapTupleGetDatum(restuple);
+
+		RelationClose(relation);
+		/* there should be only one value per oid */
+		Assert(NULL == caql_getnext(pcqCtx1)) ;
+		caql_endscan(pcqCtx1);
+
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+	else
+	{
+		caql_endscan(pcqCtx);
+		SRF_RETURN_DONE(funcctx);
+	}
+}
+
+Datum
+caql_scan_in_memory_pg_attribute(PG_FUNCTION_ARGS)
+{
+	text *tableNameText = PG_GETARG_TEXT_P(0);
+	char *tableNameStr =  text_to_cstring(tableNameText);
+
+	FuncCallContext    *funcctx;
+	Datum				result;
+	MemoryContext		oldcontext;
+	TupleDesc	tupdesc;
+	HeapTuple	restuple;
+	HeapTuple readtup = NULL;
+	cqContext  *pcqCtx;
+	StringInfoData buf;
+	initStringInfo(&buf);
+
+	Oid relid = GetRelationOid(tableNameStr);
+	if (SRF_IS_FIRSTCALL())
+	{
+		funcctx = SRF_FIRSTCALL_INIT();
+		/* switch context when allocating stuff to be used in later calls */
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+
+		tupdesc = CreateTemplateTupleDesc(NUM_COLS, false);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 1, "attname",
+						   TEXTOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 2, "atttype",
+						   OIDOID, -1, 0);
+		TupleDescInitEntry(tupdesc, (AttrNumber) 3, "atttypmod",
+						   INT4OID, -1, 0);
+		funcctx->tuple_desc = BlessTupleDesc(tupdesc);
+
+		/* create tuples for pg_attribute table */
+		pcqCtx = caql_beginscan(
+				NULL,
+				cql("SELECT * FROM pg_attribute "
+						" WHERE attrelid = :1",
+						ObjectIdGetDatum(relid)));
+		funcctx->user_fctx = pcqCtx;
+
+		/* return to original context when allocating transient memory */
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	pcqCtx = (cqContext *)funcctx->user_fctx;
+
+	if (NULL != (readtup = caql_getnext(pcqCtx)))
+	{
+		Datum values[NUM_COLS];
+		bool nulls[NUM_COLS];
+
+		Relation relation = RelationIdGetRelation(AttributeRelationId);
+		TupleDesc tupleDesc = RelationGetDescr(relation);
+
+		char *attname = DatumGetCString(tuple_getattr(readtup, tupleDesc, Anum_pg_attribute_attname));
+		text *t = cstring_to_text(attname);
+
+		values[0] = PointerGetDatum(t);
+		nulls[0]  = false;
+
+		values[1] = tuple_getattr(readtup, tupleDesc, Anum_pg_attribute_atttypid);
+		nulls[1]  = false;
+
+		values[2] = tuple_getattr(readtup, tupleDesc, Anum_pg_attribute_atttypmod);
+		nulls[2]  = false;
+		
+		/* build tuple */
+		restuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+
+		/* make the tuple into a datum */
+		result = HeapTupleGetDatum(restuple);
+
+		RelationClose(relation);
+		SRF_RETURN_NEXT(funcctx, result);
+	}
+	else
+	{
+		caql_endscan(pcqCtx);
+		SRF_RETURN_DONE(funcctx);
+	}
+}
+
+Oid GetRelationOid(char *tableName)
+{
+	cqContext *pcqCtx = caql_beginscan(
+			NULL,
+			cql("SELECT * FROM pg_class "
+					" WHERE relname = :1",
+					CStringGetDatum((char *) tableName)));
+
+	HeapTuple ht = caql_getnext(pcqCtx);
+	Oid result = InvalidOid;
+	if (NULL != ht)
+	{
+		result = HeapTupleGetOid(ht);
+	}
+
+	elog(DEBUG1, "Found relname %s, oid: %d", tableName, result);
+	caql_endscan(pcqCtx);
+	return result;
+}
+
+/*
+ * This function must be used in conjunction with
+ * caql_delete_from_heap_pg_class
+ * and exclusively for testing external Oid setting
+ */
+Datum
+caql_insert_into_heap_pg_class(PG_FUNCTION_ARGS)
+	{
+	Oid relid = PG_GETARG_OID(0);
+	char *tblname = text_to_cstring(PG_GETARG_TEXT_P(1));
+
+	Datum values[Natts_pg_class];
+	bool nulls[Natts_pg_class];
+
+	for (int i = 0; i < Natts_pg_class; i++)
+	{
+		nulls[i] = true;
+		values[i] = (Datum) 0;
+	}
+
+	NameData name;
+	namestrcpy(&name, tblname);
+
+	values[Anum_pg_class_relname - 1] = NameGetDatum(&name);
+	values[Anum_pg_class_relnamespace - 1] = ObjectIdGetDatum((Oid) NSPDBOID_CURRENT);
+	nulls[Anum_pg_class_relname - 1] = false;
+	nulls[Anum_pg_class_relnamespace - 1] = false;
+
+	cqContext  *pcqCtx = caql_beginscan(
+			NULL,
+			cql("INSERT INTO pg_class", NULL));
+
+	HeapTuple tup = caql_form_tuple(pcqCtx, values, nulls);
+	HeapTupleSetOid(tup, relid);
+
+	caql_insert(pcqCtx, tup);
+	caql_endscan(pcqCtx);
+
+	StringInfoData buf;
+	initStringInfo(&buf);
+
+	appendStringInfo(&buf, "inserted tuple to heap table pg_class (oid %u, relname %s)", relid, tblname);
+
+	PG_RETURN_TEXT_P(cstring_to_text(buf.data));
+}
+
+/*
+ * This function must be used in conjunction with
+ * caql_insert_into_heap_pg_class
+ * and exclusively for testing external Oid setting
+ */
+Datum
+caql_delete_from_heap_pg_class(PG_FUNCTION_ARGS)
+{
+	Oid relid = PG_GETARG_OID(0);
+
+	cqContext *pcqCtx = caql_beginscan(
+			NULL,
+			cql("SELECT * FROM pg_class "
+				" WHERE oid = :1 "
+				" FOR UPDATE ",
+				ObjectIdGetDatum(relid)));
+
+	HeapTuple tuple = caql_getnext(pcqCtx);
+
+	if (!HeapTupleIsValid(tuple))
+		ereport(ERROR,
+			(errcode(ERRCODE_UNDEFINED_OBJECT),
+			 errmsg("pg_class table relid=%u does not exist", relid)));
+
+	/* Delete the pg_class table entry from the catalog (pg_class) */
+	caql_delete_current(pcqCtx);
+
+	/* Finish up scan and close pg_class catalog */
+	caql_endscan(pcqCtx);
+
+	StringInfoData buf;
+	initStringInfo(&buf);
+
+	appendStringInfo(&buf, "deleted tuple oid=%u from heap table pg_class", relid);
+
+	PG_RETURN_TEXT_P(cstring_to_text(buf.data));
+}
+
+Datum
+get_next_external_oid(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_OID(GetCurrentExternalObjectId());
+}
+
+Datum
+set_next_external_oid(PG_FUNCTION_ARGS)
+{
+	Oid oid = PG_GETARG_OID(0);
+	SetCurrentExternalObjectId(oid);
+	
+	PG_RETURN_OID(GetCurrentExternalObjectId());
+}
+
+Datum
+min_external_oid(PG_FUNCTION_ARGS)
+{
+	PG_RETURN_OID(FirstExternalObjectId);
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/sql/external_oid.sql.source
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/sql/external_oid.sql.source b/src/test/feature/ExternalSource/sql/external_oid.sql.source
new file mode 100644
index 0000000..bded23e
--- /dev/null
+++ b/src/test/feature/ExternalSource/sql/external_oid.sql.source
@@ -0,0 +1,108 @@
+-- --------------------------------------
+-- test first external Oid initialization
+-- --------------------------------------
+
+-- start_matchsubs
+--                                                                                               
+-- # create a match/subs expression to handle ip addresses that change
+--
+-- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/
+-- s/oid \d+/oid SOME_OID/
+--
+-- m/.*deleted tuple oid=\d+ from heap table pg_class.*/
+-- s/oid=\d+/oid=OID/
+--
+-- end_matchsubs
+
+-- Create function that returns the first external Oid boundary
+CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid
+  AS '@SHARE_LIBRARY_PATH@', 'min_external_oid'
+  LANGUAGE C;
+
+-- Create function that returns the current external Oid
+CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid
+  AS '@SHARE_LIBRARY_PATH@', 'get_next_external_oid'
+  LANGUAGE C;
+
+-- Create function that sets the current external Oid
+CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid
+  AS '@SHARE_LIBRARY_PATH@', 'set_next_external_oid'
+  LANGUAGE C;
+
+-- Create function to insert and scan in-memory data to pg_class
+CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text
+  AS '@SHARE_LIBRARY_PATH@', 'load_json_data'
+  LANGUAGE C;
+
+-- Create function that inserts tuple with given Oid
+CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text
+  AS '@SHARE_LIBRARY_PATH@', 'caql_insert_into_heap_pg_class'
+  LANGUAGE C;
+
+-- Create function that inserts tuple with given Oid
+CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text
+  AS '@SHARE_LIBRARY_PATH@', 'caql_delete_from_heap_pg_class'
+  LANGUAGE C;
+
+-- --------------------------------------
+-- Test hcat table external oid initialization
+-- --------------------------------------
+-- Boundary should be at FirstExternalObjectId
+--SELECT min_external_oid();
+-- NextExternalObjectId is uninitialized
+SELECT get_next_external_oid();
+SELECT load_json_data('@abs_datadir@/single_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+
+BEGIN TRANSACTION;
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+-- load default.mytable -> +3 oids
+-- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype
+SELECT load_json_data('@abs_datadir@/single_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+-- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids
+-- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2)
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+END TRANSACTION;
+
+-- New transaction will reset external Oid start point
+-- Yields the same result as previous transaction
+BEGIN TRANSACTION;
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+SELECT load_json_data('@abs_datadir@/single_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+END TRANSACTION;
+
+-- --------------------------------------
+-- Test external oid rollover
+-- --------------------------------------
+BEGIN TRANSACTION;
+SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
+SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0;
+SELECT load_json_data('@abs_datadir@/multi_table.json');
+-- Used up external Oids result in Oid overflow
+SELECT get_next_external_oid();
+-- Rollover disallowed!
+SELECT load_json_data('@abs_datadir@/single_table.json');
+END TRANSACTION;
+
+-- --------------------------------------
+-- Test external Oid boundary
+-- --------------------------------------
+-- Create a tuple with Oid larger than FirstExternalObjectId
+-- Will fail during next session when try to query HCatalog
+-- Because external Oid boundary is violated
+SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl');
+
+-- cleanup
+SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20);
+
+DROP FUNCTION caql_delete_from_heap_pg_class(relid oid);
+DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text);
+DROP FUNCTION load_json_data(filename text);
+DROP FUNCTION get_next_external_oid();
+DROP FUNCTION set_next_external_oid(ext_oid oid);
+DROP FUNCTION min_external_oid();

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/ExternalSource/test_external_oid.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ExternalSource/test_external_oid.cpp b/src/test/feature/ExternalSource/test_external_oid.cpp
new file mode 100644
index 0000000..3326514
--- /dev/null
+++ b/src/test/feature/ExternalSource/test_external_oid.cpp
@@ -0,0 +1,35 @@
+#include <string>
+#include <iostream>
+#include "gtest/gtest.h"
+
+#include "lib/sql_util.h"
+#include "lib/file_replace.h"
+
+using hawq::test::SQLUtility;
+using hawq::test::FileReplace;
+
+class TestExternalOid : public ::testing::Test {
+ public:
+  TestExternalOid() {}
+  ~TestExternalOid() {}
+};
+
+TEST_F(TestExternalOid, TestExternalOidAll) {
+  SQLUtility util;
+  FileReplace frep;
+  auto test_root = util.getTestRootPath();
+  std::cout << test_root << std::endl;
+
+  std::unordered_map<std::string, std::string> D;
+  D["@SHARE_LIBRARY_PATH@"] = test_root + "/ExternalSource/lib/function.so";
+  D["@abs_datadir@"] = test_root + "/ExternalSource/data";
+  frep.replace(test_root + "/ExternalSource/sql/external_oid.sql.source",
+               test_root + "/ExternalSource/sql/external_oid.sql",
+               D);
+  frep.replace(test_root + "/ExternalSource/ans/external_oid.ans.source",
+               test_root + "/ExternalSource/ans/external_oid.ans",
+               D);
+ 
+  util.execSQLFile("ExternalSource/sql/external_oid.sql",
+                   "ExternalSource/ans/external_oid.ans");
+}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/feature/Makefile
----------------------------------------------------------------------
diff --git a/src/test/feature/Makefile b/src/test/feature/Makefile
index adc6acc..f393a76 100644
--- a/src/test/feature/Makefile
+++ b/src/test/feature/Makefile
@@ -27,9 +27,11 @@ all: $(OBJS) sharelib
 
 sharelib:
 	cd UDF/lib || exit 1; $(MAKE) || exit 2
+	cd ExternalSource/lib || exit 1; $(MAKE) || exit 2
 
 sharelibclean:
 	cd UDF/lib || exit 1; $(RM) *.o *.so || exit 2
+	cd ExternalSource/lib || exit 1; $(MAKE) || exit 2
 
 doc:
 	doxygen doxygen_template

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/regress/input/external_oid.source
----------------------------------------------------------------------
diff --git a/src/test/regress/input/external_oid.source b/src/test/regress/input/external_oid.source
deleted file mode 100644
index bd4724a..0000000
--- a/src/test/regress/input/external_oid.source
+++ /dev/null
@@ -1,118 +0,0 @@
--- --------------------------------------
--- test first external Oid initialization
--- --------------------------------------
-
--- start_matchsubs
---                                                                                               
--- # create a match/subs expression to handle ip addresses that change
---
--- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/
--- s/oid \d+/oid SOME_OID/
---
--- m/.*deleted tuple oid=\d+ from heap table pg_class.*/
--- s/oid=\d+/oid=OID/
---
--- end_matchsubs
-
--- Create function that returns the first external Oid boundary
-CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'min_external_oid'
-  LANGUAGE C;
-
--- Create function that returns the current external Oid
-CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'get_next_external_oid'
-  LANGUAGE C;
-
--- Create function that sets the current external Oid
-CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'set_next_external_oid'
-  LANGUAGE C;
-
--- Create function to insert and scan in-memory data to pg_class
-CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'load_json_data'
-  LANGUAGE C;
-
--- Create function that inserts tuple with given Oid
-CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_insert_into_heap_pg_class'
-  LANGUAGE C;
-
--- Create function that inserts tuple with given Oid
-CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_delete_from_heap_pg_class'
-  LANGUAGE C;
-
--- --------------------------------------
--- Test hcat table external oid initialization
--- --------------------------------------
--- Boundary should be at FirstExternalObjectId
---SELECT min_external_oid();
--- NextExternalObjectId is uninitialized
-SELECT get_next_external_oid();
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
--- load default.mytable -> +3 oids
--- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
--- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids
--- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2)
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-END TRANSACTION;
-
--- New transaction will reset external Oid start point
--- Yields the same result as previous transaction
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-END TRANSACTION;
-
--- --------------------------------------
--- Test external oid rollover
--- --------------------------------------
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
-SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0;
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
--- Used up external Oids result in Oid overflow
-SELECT get_next_external_oid();
--- Rollover disallowed!
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-END TRANSACTION;
-
--- --------------------------------------
--- Test external Oid boundary
--- --------------------------------------
--- Create a tuple with Oid larger than FirstExternalObjectId
--- Will fail during next session when try to query HCatalog
--- Because external Oid boundary is violated
-SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl');
-
--- Fresh session
-\c
--- NextExternalObjectId is uninitialized
-SELECT get_next_external_oid();
--- Should fail
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
--- NextExternalObjectId is still uninitialized
-SELECT get_next_external_oid();
-
--- cleanup
-SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20);
-
-DROP FUNCTION caql_delete_from_heap_pg_class(relid oid);
-DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text);
-DROP FUNCTION load_json_data(filename text);
-DROP FUNCTION get_next_external_oid();
-DROP FUNCTION set_next_external_oid(ext_oid oid);
-DROP FUNCTION min_external_oid();
-

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/regress/known_good_schedule
----------------------------------------------------------------------
diff --git a/src/test/regress/known_good_schedule b/src/test/regress/known_good_schedule
index 0cd48f3..1f95e49 100755
--- a/src/test/regress/known_good_schedule
+++ b/src/test/regress/known_good_schedule
@@ -202,5 +202,4 @@ ignore: co_disabled
 test: caqlinmem
 test: hcatalog_lookup
 test: json_load
-test: external_oid
 test: validator_function

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bab04b57/src/test/regress/output/external_oid.source
----------------------------------------------------------------------
diff --git a/src/test/regress/output/external_oid.source b/src/test/regress/output/external_oid.source
deleted file mode 100644
index d0a3730..0000000
--- a/src/test/regress/output/external_oid.source
+++ /dev/null
@@ -1,211 +0,0 @@
--- --------------------------------------
--- test first external Oid initialization
--- --------------------------------------
--- start_matchsubs
---                                                                                               
--- # create a match/subs expression to handle ip addresses that change
---
--- m/.*inserted tuple to heap table pg_class \(oid \d+, relname table_xl\).*/
--- s/oid \d+/oid SOME_OID/
---
--- m/.*deleted tuple oid=\d+ from heap table pg_class.*/
--- s/oid=\d+/oid=OID/
---
--- end_matchsubs
--- Create function that returns the first external Oid boundary
-CREATE OR REPLACE FUNCTION min_external_oid() RETURNS oid
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'min_external_oid'
-  LANGUAGE C;
--- Create function that returns the current external Oid
-CREATE OR REPLACE FUNCTION get_next_external_oid() RETURNS oid
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'get_next_external_oid'
-  LANGUAGE C;
--- Create function that sets the current external Oid
-CREATE OR REPLACE FUNCTION set_next_external_oid(ext_oid oid) RETURNS oid
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'set_next_external_oid'
-  LANGUAGE C;
--- Create function to insert and scan in-memory data to pg_class
-CREATE OR REPLACE FUNCTION load_json_data(filename text) RETURNS text
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'load_json_data'
-  LANGUAGE C;
--- Create function that inserts tuple with given Oid
-CREATE OR REPLACE FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text) RETURNS text
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_insert_into_heap_pg_class'
-  LANGUAGE C;
--- Create function that inserts tuple with given Oid
-CREATE OR REPLACE FUNCTION caql_delete_from_heap_pg_class(relid oid) RETURNS text
-  AS '@abs_builddir@/regress@DLSUFFIX@', 'caql_delete_from_heap_pg_class'
-  LANGUAGE C;
--- --------------------------------------
--- Test hcat table external oid initialization
--- --------------------------------------
--- Boundary should be at FirstExternalObjectId
---SELECT min_external_oid();
--- NextExternalObjectId is uninitialized
-SELECT get_next_external_oid();
- get_next_external_oid 
------------------------
-                     0
-(1 row)
-
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-  load_json_data  
-------------------
- default.mytable 
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column? 
-----------
-        0
-(1 row)
-
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column? 
-----------
-        0
-(1 row)
-
--- load default.mytable -> +3 oids
--- 1 oid for namespace 'default', 1 oid for relation 'mytable', 1 oid for reltype
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-  load_json_data  
-------------------
- default.mytable 
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column? 
-----------
-        3
-(1 row)
-
--- load db1.ht1, db2.ht1, db2.ht2 -> +8 oids
--- oids: db1, ht1(db1), db2, ht1(db2), ht2, reltype(ht1, ht1, ht2)
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
-      load_json_data      
---------------------------
- db1.ht1 db2.ht1 db2.ht2 
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column? 
-----------
-       11
-(1 row)
-
-END TRANSACTION;
--- New transaction will reset external Oid start point
--- Yields the same result as previous transaction
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column? 
-----------
-        0
-(1 row)
-
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-  load_json_data  
-------------------
- default.mytable 
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column? 
-----------
-        3
-(1 row)
-
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
-      load_json_data      
---------------------------
- db1.ht1 db2.ht1 db2.ht2 
-(1 row)
-
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column? 
-----------
-       11
-(1 row)
-
-END TRANSACTION;
--- --------------------------------------
--- Test external oid rollover
--- --------------------------------------
-BEGIN TRANSACTION;
-SELECT get_next_external_oid()::bigint - min_external_oid()::bigint;
- ?column? 
-----------
-        0
-(1 row)
-
-SELECT set_next_external_oid( oid(min_external_oid()::bigint + (10*power(2,20))::bigint - 8 + 1) ) > 0;
- ?column? 
-----------
- t
-(1 row)
-
-SELECT load_json_data('@abs_builddir@/data/hcatalog/multi_table.json');
-      load_json_data      
---------------------------
- db1.ht1 db2.ht1 db2.ht2 
-(1 row)
-
--- Used up external Oids result in Oid overflow
-SELECT get_next_external_oid();
- get_next_external_oid
------------------------
-                     0
-(1 row)
-
--- Rollover disallowed!
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-ERROR:  number of external objects from HCatalog exceeded 10M during transaction
-HINT:  Separate HCatalog queries into different transactions to process.
-END TRANSACTION;
--- --------------------------------------
--- Test external Oid boundary
--- --------------------------------------
--- Create a tuple with Oid larger than FirstExternalObjectId
--- Will fail during next session when try to query HCatalog
--- Because external Oid boundary is violated
-SELECT caql_insert_into_heap_pg_class(min_external_oid()::bigint + 20, 'table_xl');
-                      caql_insert_into_heap_pg_class                      
---------------------------------------------------------------------------
- inserted tuple to heap table pg_class (oid 4293918750, relname table_xl)
-(1 row)
-
--- Fresh session
-\c
--- NextExternalObjectId is uninitialized
-SELECT get_next_external_oid();
- get_next_external_oid 
------------------------
-                     0
-(1 row)
-
--- Should fail
-SELECT load_json_data('@abs_builddir@/data/hcatalog/single_table.json');
-ERROR:  database does not have enough available Oids to support HCatalog queries
-HINT:  Database VACUUM may recycle unused Oids.
--- NextExternalObjectId is still uninitialized
-SELECT get_next_external_oid();
- get_next_external_oid 
------------------------
-                     0
-(1 row)
-
--- cleanup
-SELECT caql_delete_from_heap_pg_class(min_external_oid()::bigint + 20);
-            caql_delete_from_heap_pg_class             
--------------------------------------------------------
- deleted tuple oid=4293918750 from heap table pg_class
-(1 row)
-
-DROP FUNCTION caql_delete_from_heap_pg_class(relid oid);
-DROP FUNCTION caql_insert_into_heap_pg_class(relid oid, tblname text);
-DROP FUNCTION load_json_data(filename text);
-DROP FUNCTION get_next_external_oid();
-DROP FUNCTION set_next_external_oid(ext_oid oid);
-DROP FUNCTION min_external_oid();


Mime
View raw message