hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From y...@apache.org
Subject incubator-hawq git commit: HAWQ-128. Cleanup pg_resqueue impact in catalog and dump corresponding components
Date Fri, 06 Nov 2015 06:38:24 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 16328bfcf -> 185b83cd6


HAWQ-128. Cleanup pg_resqueue impact in catalog and dump corresponding components


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/185b83cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/185b83cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/185b83cd

Branch: refs/heads/master
Commit: 185b83cd6969997adb2b062f01df57e01bda2b42
Parents: 16328bf
Author: Yi Jin <yjin@pivotal.io>
Authored: Fri Nov 6 14:37:48 2015 +0800
Committer: Yi Jin <yjin@pivotal.io>
Committed: Fri Nov 6 14:37:48 2015 +0800

----------------------------------------------------------------------
 src/backend/catalog/caql/caqlaccess.c           |  9 --
 src/backend/catalog/system_views.sql            | 18 ++--
 src/backend/resourcemanager/resqueuemanager.c   | 61 ++++++++++++--
 src/bin/pg_dump/pg_dumpall.c                    | 88 +++++++++++---------
 .../regress/expected/validator_function.out     | 12 +--
 src/tools/pgindent/typedefs.list                |  2 -
 tools/bin/gpdetective                           |  2 +-
 tools/bin/gppylib/gpcatalog.py                  |  2 -
 tools/bin/lib/gpcheckcat                        | 27 ------
 9 files changed, 119 insertions(+), 102 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/185b83cd/src/backend/catalog/caql/caqlaccess.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/caql/caqlaccess.c b/src/backend/catalog/caql/caqlaccess.c
index 4b7ae49..28f0ec0 100644
--- a/src/backend/catalog/caql/caqlaccess.c
+++ b/src/backend/catalog/caql/caqlaccess.c
@@ -939,15 +939,6 @@ caql_getattr(cqContext *pCtx, AttrNumber attnum, bool *isnull)
 	Assert(HeapTupleIsValid(pCtx->cq_lasttup));
 
 	return caql_getattr_internal(pCtx, pCtx->cq_lasttup, attnum, isnull);
-
-	/*
-	 * NOTE: could this be used if caql is extended to support joins, eg
-	 * what would attnum be for
-	 * "SELECT * FROM pg_resqueue, pg_resqueuecapability ..." ?
-	 * Potentially, the attnum is just the ordinal position of the combined
-	 * SELECT list, eg you could reference pg_resqueuecapability.restypid
-	 * as (Natts_pg_resqueue+Anum_pg_resourcetype_restypid).
-	 */
 }
 
 /* ----------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/185b83cd/src/backend/catalog/system_views.sql
----------------------------------------------------------------------
diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql
index 9e4eee9..8ba0a51 100644
--- a/src/backend/catalog/system_views.sql
+++ b/src/backend/catalog/system_views.sql
@@ -371,15 +371,15 @@ CREATE VIEW pg_stat_database AS
             pg_stat_get_db_blocks_hit(D.oid) AS blks_hit 
     FROM pg_database D;
 
-CREATE VIEW pg_stat_resqueues AS
-	SELECT
-		Q.oid AS queueid,
-		Q.rsqname AS queuename,
-		pg_stat_get_queue_num_exec(Q.oid) AS n_queries_exec,
-		pg_stat_get_queue_num_wait(Q.oid) AS n_queries_wait,
-		pg_stat_get_queue_elapsed_exec(Q.oid) AS elapsed_exec,
-		pg_stat_get_queue_elapsed_wait(Q.oid) AS elapsed_wait
-	FROM pg_resqueue AS Q;
+-- CREATE VIEW pg_stat_resqueues AS
+--	SELECT
+--		Q.oid AS queueid,
+--		Q.rsqname AS queuename,
+--		pg_stat_get_queue_num_exec(Q.oid) AS n_queries_exec,
+--		pg_stat_get_queue_num_wait(Q.oid) AS n_queries_wait,
+--		pg_stat_get_queue_elapsed_exec(Q.oid) AS elapsed_exec,
+--		pg_stat_get_queue_elapsed_wait(Q.oid) AS elapsed_wait
+--	FROM pg_resqueue AS Q;
 
 -- Resource queue views
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/185b83cd/src/backend/resourcemanager/resqueuemanager.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/resqueuemanager.c b/src/backend/resourcemanager/resqueuemanager.c
index 4315daf..4f38a65 100644
--- a/src/backend/resourcemanager/resqueuemanager.c
+++ b/src/backend/resourcemanager/resqueuemanager.c
@@ -860,6 +860,23 @@ int updateResourceQueueAttributes(List 			 	*attributes,
 						 queue->ResourceOvercommit);
 			break;
 
+		case RSQ_TBL_ATTR_ALLOCATION_POLICY:
+			res = SimpleStringToMapIndexInt8(
+						attrvalue,
+						(char *)RSQDDLValueAllocationPolicy,
+						RSQ_ALLOCATION_POLICY_COUNT,
+						sizeof(RSQDDLValueAllocationPolicy[0]),
+						&(queue->AllocatePolicy));
+			if ( res != FUNC_RETURN_OK )
+			{
+				snprintf(errorbuf, errorbufsize,
+						 "Allocation policy %s is not valid.",
+						 attrvalue->Str);
+				ELOG_ERRBUF_MESSAGE(WARNING, errorbuf)
+				return res;
+			}
+			break;
+
 		case RSQ_DDL_ATTR_NVSEG_UPPER_LIMIT:
 			res = SimpleStringToInt32(attrvalue, &(queue->NVSegUpperLimit));
 			if ( res != FUNC_RETURN_OK )
@@ -875,22 +892,50 @@ int updateResourceQueueAttributes(List 			 	*attributes,
 						 "updated virtual segment size upper limit %d",
 						 queue->NVSegUpperLimit);
 			break;
+		case RSQ_DDL_ATTR_NVSEG_LOWER_LIMIT:
+			res = SimpleStringToInt32(attrvalue, &(queue->NVSegLowerLimit));
+			if ( res != FUNC_RETURN_OK )
+			{
+				snprintf(errorbuf, errorbufsize,
+						 "Virtual segment lower limit %s is not valid.",
+						 attrvalue->Str);
+				ELOG_ERRBUF_MESSAGE(WARNING, errorbuf)
+				return res;
+			}
+
+			elog(DEBUG3, "Resource manager updateResourceQueueAttributes() "
+						 "updated virtual segment size lower limit %d",
+						 queue->NVSegLowerLimit);
+			break;
+		case RSQ_DDL_ATTR_NVSEG_UPPER_LIMIT_PERSEG:
+			res = SimpleStringToDouble(attrvalue, &(queue->NVSegUpperLimitPerSeg));
+			if ( res != FUNC_RETURN_OK )
+			{
+				snprintf(errorbuf, errorbufsize,
+						 "Virtual segment upper limit per segment %s is not valid.",
+						 attrvalue->Str);
+				ELOG_ERRBUF_MESSAGE(WARNING, errorbuf)
+				return res;
+			}
 
-		case RSQ_TBL_ATTR_ALLOCATION_POLICY:
-			res = SimpleStringToMapIndexInt8(
-						attrvalue,
-						(char *)RSQDDLValueAllocationPolicy,
-						RSQ_ALLOCATION_POLICY_COUNT,
-						sizeof(RSQDDLValueAllocationPolicy[0]),
-						&(queue->AllocatePolicy));
+			elog(DEBUG3, "Resource manager updateResourceQueueAttributes() "
+						 "updated virtual segment size upper limit per segment %lf",
+						 queue->NVSegUpperLimitPerSeg);
+			break;
+		case RSQ_DDL_ATTR_NVSEG_LOWER_LIMIT_PERSEG:
+			res = SimpleStringToDouble(attrvalue, &(queue->NVSegLowerLimitPerSeg));
 			if ( res != FUNC_RETURN_OK )
 			{
 				snprintf(errorbuf, errorbufsize,
-						 "Allocation policy %s is not valid.",
+						 "Virtual segment lower limit per segment %s is not valid.",
 						 attrvalue->Str);
 				ELOG_ERRBUF_MESSAGE(WARNING, errorbuf)
 				return res;
 			}
+
+			elog(DEBUG3, "Resource manager updateResourceQueueAttributes() "
+						 "updated virtual segment size lower limit per segment %lf",
+						 queue->NVSegLowerLimitPerSeg);
 			break;
 
 		case RSQ_TBL_ATTR_STATUS:

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/185b83cd/src/bin/pg_dump/pg_dumpall.c
----------------------------------------------------------------------
diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c
index fdbbd46..c700b5f 100644
--- a/src/bin/pg_dump/pg_dumpall.c
+++ b/src/bin/pg_dump/pg_dumpall.c
@@ -565,47 +565,50 @@ dumpResQueues(PGconn *conn)
 	bool		bWith = false;
 
 	printfPQExpBuffer(buf,
-					  "SELECT rsqname, 'parent' as resname, "
-					  "rsq_parent::text as ressetting, "
+					  "SELECT oid, rsqname, 'parent' as resname, "
+					  "parentoid::text as ressetting, "
 					  "1 as ord FROM pg_resqueue "
 					  "UNION "
-					  "SELECT rsqname, 'active_statements' as resname, "
-					  "rsq_active_stats_cluster::text as ressetting, "
+					  "SELECT oid, rsqname, 'active_statements' as resname, "
+					  "activestats::text as ressetting, "
 					  "2 as ord FROM pg_resqueue "
 					  "UNION "
-					  "SELECT rsqname, 'memory_limit_cluster' as resname, "
-					  "rsq_memory_limit_cluster::text as ressetting, "
+					  "SELECT oid, rsqname, 'memory_limit_cluster' as resname, "
+					  "memorylimit::text as ressetting, "
 					  "3 as ord FROM pg_resqueue "
 					  "UNION "
-					  "SELECT  rsqname, 'core_limit_cluster' as resname, "
-					  "rsq_core_limit_cluster::text as ressetting, "
+					  "SELECT oid, rsqname, 'core_limit_cluster' as resname, "
+					  "corelimit::text as ressetting, "
 					  "4 as ord FROM pg_resqueue "
 					  "UNION "
-					  "SELECT rsqname, 'resource_upper_factor' as resname, "
-					  "rsq_resource_upper_factor::text as ressetting, "
+					  "SELECT oid, rsqname, 'resource_overcommit_factor' as resname, "
+					  "resovercommit::text as ressetting, "
 					  "5 as ord FROM pg_resqueue "
 					  "UNION "
-					  "SELECT rsqname, 'allocation_policy' as resname, "
-					  "rsq_allocation_policy::text as ressetting, "
+					  "SELECT oid, rsqname, 'allocation_policy' as resname, "
+					  "allocpolicy::text as ressetting, "
 					  "6 as ord FROM pg_resqueue "
 					  "UNION "
-					  "SELECT rsqname, 'vseg_resource_quota' as resname, "
-					  "rsq_vseg_resource_quota::text as ressetting, "
+					  "SELECT oid, rsqname, 'vseg_resource_quota' as resname, "
+					  "vsegresourcequota::text as ressetting, "
 					  "7 as ord FROM pg_resqueue "
 					  "UNION "
-					  "SELECT rsqname, 'vsegment_upper_limit' as resname, "
-					  "rsq_vseg_upper_limit::text as ressetting, "
+					  "SELECT oid, rsqname, 'nvseg_upper_limit' as resname, "
+					  "nvsegupperlimit::text as ressetting, "
 					  "8 as ord FROM pg_resqueue "
-					  "%s"
-					  "order by rsqname,  ord",
-					  (server_version >= 80214 ?
-					   "UNION "
-					   "select rq.rsqname ,  rt.resname,  rc.ressetting, "
-					   "rt.restypid as ord from "
-					   "pg_resqueue rq,  pg_resourcetype rt, "
-					   "pg_resqueuecapability rc where "
-					   "rq.oid=rc.resqueueid and rc.restypid = rt.restypid "
-					   : "")
+					  "UNION "
+					  "SELECT oid, rsqname, 'nvseg_lower_limit' as resname, "
+					  "nvseglowerlimit::text as ressetting, "
+					  "9 as ord FROM pg_resqueue "
+					  "UNION "
+					  "SELECT oid, rsqname, 'nvseg_upper_limit_perseg' as resname, "
+					  "nvsegupperlimitperseg::text as ressetting, "
+					  "10 as ord FROM pg_resqueue "
+					  "UNION "
+					  "SELECT oid, rsqname, 'nvseg_lower_limit_perseg' as resname, "
+					  "nvseglowerlimitperseg::text as ressetting, "
+					  "11 as ord FROM pg_resqueue "
+					  "order by oid, ord"
 		);
 
 	res = executeQuery(conn, buf->data);
@@ -617,28 +620,31 @@ dumpResQueues(PGconn *conn)
 	if (PQntuples(res) > 0)
 	    fprintf(OPF, "--\n-- Resource Queues\n--\n\n");
 
-
 	/*
 	 * settings for resource queue are spread over multiple rows, but sorted
-	 * by queue name (and ranked in order of resname ) eg:
+	 * by queue oid :
 	 *
-	 * rsqname	  |		resname		| ressetting | ord
-	 * -----------+-----------------+------------+-----
+	 * oid | rsqname   |     resname     | ressetting | ord
+	 * ----+-----------+-----------------+------------+-----
 	 *
-	 * This format lets us support an arbitrary number of resqueuecapability
+	 * This format lets us support an arbitrary number of capability
 	 * entries.  So watch for change of rsqname to switch to next CREATE
 	 * statement.
 	 *
+	 * We order rows based on oid values, because we would like to let resource
+	 * queues be created from the ones having smaller oid values, which guarantees
+	 * that every queue can find its parent queue.
+	 *
 	 */
 
 	for (i = 0; i < PQntuples(res); i++)
 	{
-		const char *rsqname;
-		const char *resname;
-		const char *ressetting;
+		const char *rsqname		= NULL;
+		const char *resname		= NULL;
+		const char *ressetting	= NULL;
 
-		rsqname = PQgetvalue(res, i, i_rsqname);
-		resname = PQgetvalue(res, i, i_resname);
+		rsqname    = PQgetvalue(res, i, i_rsqname);
+		resname    = PQgetvalue(res, i, i_resname);
 		ressetting = PQgetvalue(res, i, i_ressetting);
 
 		/* skip pg_root */
@@ -688,10 +694,16 @@ dumpResQueues(PGconn *conn)
 			appendPQExpBuffer(buf, "\n WITH (");
 		}
 
-		if (0 == strcmp("active_statements", resname) || 0 == strcmp("vsegment_upper_limit", resname))
+		if (0 == strcmp("active_statements",          resname) ||
+			0 == strcmp("resource_overcommit_factor", resname) ||
+			0 == strcmp("nvseg_upper_limit",          resname) ||
+			0 == strcmp("nvseg_lower_limit",          resname) ||
+			0 == strcmp("nvseg_upper_limit_perseg",   resname) ||
+			0 == strcmp("nvseg_lower_limit_perseg",   resname))
 			/* numeric */
 			appendPQExpBuffer(buf, " %s=%s", resname, ressetting);
-		else if (0 == strcmp("parent", resname)) {
+		else if (0 == strcmp("parent", resname))
+		{
 			/* find parent's name with oid. */
 			appendPQExpBuffer(sql, "SELECT rsqname FROM pg_resqueue WHERE oid='%s'", ressetting);
 			res2 = executeQuery(conn, sql->data);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/185b83cd/src/test/regress/expected/validator_function.out
----------------------------------------------------------------------
diff --git a/src/test/regress/expected/validator_function.out b/src/test/regress/expected/validator_function.out
index c5a5296..3e5aedf 100644
--- a/src/test/regress/expected/validator_function.out
+++ b/src/test/regress/expected/validator_function.out
@@ -20,14 +20,14 @@ REVOKE EXECUTE ON FUNCTION foo() FROM user1;
 COMMIT;
 -- DEBUG info
 SELECT usesuper FROM pg_user WHERE usename = current_user;
- usesuper
+ usesuper 
 ----------
  t
 (1 row)
 
 -- foo() works
 SELECT foo();
- foo
+ foo 
 -----
  t
 (1 row)
@@ -42,9 +42,9 @@ FROM (
   SELECT oid FROM pg_proc
   WHERE proname = 'foo'
 ) AS bar;
- plpgsql_validator
+ plpgsql_validator 
 -------------------
-
+ 
 (1 row)
 
 -- Wrong validator of a different language fails on foo()
@@ -53,7 +53,7 @@ FROM (
   SELECT oid FROM pg_proc
   WHERE proname = 'foo'
 ) AS bar;
-ERROR:  language validation function 2248 called for language 10889 in stead of 10888
+ERROR:  language validation function 2248 called for language 10886 in stead of 10885
 -- --------------------------------------
 -- Cannot run validator on functions to
 -- which user has no privilege to execute
@@ -61,7 +61,7 @@ ERROR:  language validation function 2248 called for language 10889 in stead
of
 SET SESSION AUTHORIZATION user1;
 -- DEBUG info
 SELECT usesuper FROM pg_user WHERE usename = 'user1' AND current_user = 'user1';
- usesuper
+ usesuper 
 ----------
  f
 (1 row)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/185b83cd/src/tools/pgindent/typedefs.list
----------------------------------------------------------------------
diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list
index d31b74e..0359d2b 100644
--- a/src/tools/pgindent/typedefs.list
+++ b/src/tools/pgindent/typedefs.list
@@ -844,7 +844,6 @@ FormData_pg_partition
 FormData_pg_partition_rule
 FormData_pg_pltemplate
 FormData_pg_proc
-FormData_pg_resourcetype
 FormData_pg_resqueue
 FormData_pg_rewrite
 FormData_pg_sequence
@@ -900,7 +899,6 @@ Form_pg_partition
 Form_pg_partition_rule
 Form_pg_pltemplate
 Form_pg_proc
-Form_pg_resourcetype
 Form_pg_resqueue
 Form_pg_rewrite
 Form_pg_sequence

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/185b83cd/tools/bin/gpdetective
----------------------------------------------------------------------
diff --git a/tools/bin/gpdetective b/tools/bin/gpdetective
index 3d091ef..d336391 100755
--- a/tools/bin/gpdetective
+++ b/tools/bin/gpdetective
@@ -720,7 +720,7 @@ class GPDetective:
 
         self.writeSelect('pg_stat_resqueue.log',
         '''
-            SELECT * FROM pg_catalog.pg_stat_resqueues            
+            SELECT * FROM pg_catalog.pg_resqueue_status            
         ''')
         
         self.writeSelect('pg_stat_database.log',

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/185b83cd/tools/bin/gppylib/gpcatalog.py
----------------------------------------------------------------------
diff --git a/tools/bin/gppylib/gpcatalog.py b/tools/bin/gppylib/gpcatalog.py
index 88ea72f..2d2c6a9 100644
--- a/tools/bin/gppylib/gpcatalog.py
+++ b/tools/bin/gppylib/gpcatalog.py
@@ -364,11 +364,9 @@ class GPCatalog():
             
             # MPP-11858: pg_resqueue/pg_resqueuecapability oid inconsistencies
             self._tables['pg_resqueue']._setKnownDifferences("oid")
-            self._tables['pg_resqueuecapability']._setKnownDifferences("oid")
 
             # pg_resqueue is master only in 4.0
             self._tables['pg_resqueue']._setMasterOnly()
-            self._tables['pg_resqueuecapability']._setMasterOnly()
 
 
     def _validate(self):

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/185b83cd/tools/bin/lib/gpcheckcat
----------------------------------------------------------------------
diff --git a/tools/bin/lib/gpcheckcat b/tools/bin/lib/gpcheckcat
index ac59ebb..5c602d1 100755
--- a/tools/bin/lib/gpcheckcat
+++ b/tools/bin/lib/gpcheckcat
@@ -3972,30 +3972,6 @@ def getClassOidForRelfilenode(relfilenode):
         setError(ERROR_NOREPAIR)
         myprint('  Execution error: ' + str(e))
 
-
-#-------------------------------------------------------------------------------
-def getResourceTypeOid(oid):
-    qry = """ 
-          SELECT oid
-          FROM ( 
-               SELECT oid FROM pg_resourcetype WHERE restypid = %d
-               UNION ALL
-               SELECT oid FROM gp_dist_random('pg_resourcetype')
-                WHERE restypid = %d
-          ) alloids
-          GROUP BY oid ORDER BY count(*) desc LIMIT 1
-          """ % (oid, oid)
-
-    try:
-        db = connect()
-        curs = db.query(qry)
-        if len(curs.dictresult()) == 0: return 0
-        return curs.dictresult().pop()['oid']
-    except Exception, e:
-        setError(ERROR_NOREPAIR)
-        myprint('  Execution error: ' + str(e))
-
-  
 #-------------------------------------------------------------------------------
 # Process results of tests (CC and FK) for a particular catalog:
 #   - Categorize entry into GPObject
@@ -4158,12 +4134,9 @@ def processForeignKeyResult(catname, pkcatname, colname, allValues):
 
         # Special cases:
         #    1. pg_class(oid) referencing pg_type(oid) - relation & composite type
-        #    2. pg_resqueuecapability(restypid) referencing pg_resourcetype(restypid)
         if pkcatname == 'pg_type' and fkeytab == 'pg_class':
             rowObjName = 'pg_class'
             oid = getOidFromPK(rowObjName, fkeys)
-        elif pkcatname == 'pg_resourcetype' and gpColName == 'restypid':
-            oid = getResourceTypeOid(oid)
         gpObj = getGPObject(oid, rowObjName)
         gpObj.setForeignKeyIssue(issue)
 


Mime
View raw message