Return-Path: X-Original-To: apmail-hawq-commits-archive@minotaur.apache.org Delivered-To: apmail-hawq-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ED1DB18B94 for ; Fri, 16 Oct 2015 01:08:56 +0000 (UTC) Received: (qmail 26715 invoked by uid 500); 16 Oct 2015 01:08:51 -0000 Delivered-To: apmail-hawq-commits-archive@hawq.apache.org Received: (qmail 26677 invoked by uid 500); 16 Oct 2015 01:08:51 -0000 Mailing-List: contact commits-help@hawq.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hawq.incubator.apache.org Delivered-To: mailing list commits@hawq.incubator.apache.org Received: (qmail 26668 invoked by uid 99); 16 Oct 2015 01:08:51 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 Oct 2015 01:08:51 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 52E48C6169 for ; Fri, 16 Oct 2015 01:08:51 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.801 X-Spam-Level: * X-Spam-Status: No, score=1.801 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, T_FILL_THIS_FORM_SHORT=0.01, T_RP_MATCHES_RCVD=-0.01, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-eu-west.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id MiutjQ0w1kir for ; Fri, 16 Oct 2015 01:08:38 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-eu-west.apache.org (ASF Mail Server at mx1-eu-west.apache.org) with SMTP id 7A6F021277 for ; Fri, 16 Oct 2015 01:08:36 +0000 (UTC) Received: (qmail 25746 invoked by uid 99); 16 Oct 2015 01:08:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 Oct 2015 01:08:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 7F4B6E111F; Fri, 16 Oct 2015 01:08:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: yjin@apache.org To: commits@hawq.incubator.apache.org Date: Fri, 16 Oct 2015 01:08:35 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/4] incubator-hawq git commit: HAWQ-25. Add resource queue new ddl statement implementation, refine partial GUC variable names, use libyarn supporting kerberos. Repository: incubator-hawq Updated Branches: refs/heads/master 7e6838f84 -> a413a4265 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a413a426/src/include/catalog/pg_resqueue.h ---------------------------------------------------------------------- diff --git a/src/include/catalog/pg_resqueue.h b/src/include/catalog/pg_resqueue.h index 41961cb..38bb6a6 100644 --- a/src/include/catalog/pg_resqueue.h +++ b/src/include/catalog/pg_resqueue.h @@ -28,17 +28,20 @@ with (camelcase=ResQueue, shared=true, relid=6026, reltype_oid=9830, toast_oid=9820, toast_index=9821, toast_reltype=9822) ( rsqname name, -- name of resource queue - rsq_parent oid, -- oid of resource queue - rsq_active_stats_cluster integer, -- active statement count limit - rsq_memory_limit_cluster text, -- memory limit in cluster - rsq_core_limit_cluster text, -- core limit in cluster - rsq_resource_upper_factor real, -- resource upper limit in cluster - rsq_allocation_policy text, -- query resource allocation policy - rsq_vseg_resource_quota text, -- vsegment resource quota - rsq_vseg_upper_limit integer, -- vsegment size upper limit - rsq_creation_time timestamp with time zone, -- when the queue is created - rsq_update_time timestamp with time zone, -- when the queue is updated ( create or alter ) - rsq_status text, -- the status of resource queue. + parentoid oid, -- oid of resource queue + activestats integer, -- active statement count limit + memorylimit text, -- memory limit in cluster + corelimit text, -- core limit in cluster + resovercommit real, -- resource upper limit in cluster + allocpolicy text, -- query resource allocation policy + vsegresourcequota text, -- vsegment resource quota + nvsegupperlimit integer, -- vsegment size upper limit + nvseglowerlimit integer, -- vsegment size lower limit + nvsegupperlimitperseg real, -- vsegment size upper limit per segment + nvseglowerlimitperseg real, -- vsegment size lower limit per segment + creationtime timestamp with time zone, -- when the queue is created + updatetime timestamp with time zone, -- when the queue is updated ( create or alter ) + status text, -- the status of resource queue ); create unique index on pg_resqueue(oid) with (indexid=6027); @@ -51,7 +54,7 @@ WARNING: DO NOT MODIFY THE FOLLOWING SECTION: Generated by ./tidycat.pl version 34 - on Thu Aug 7 17:27:40 2014 + on Thu Sep 17 15:51:40 2015 */ @@ -66,7 +69,7 @@ */ /* - * The CATALOG definition has to refer to the type of "rsq_creation_time" et al as + * The CATALOG definition has to refer to the type of "creationtime" et al as * "timestamptz" (lower case) so that bootstrap mode recognizes it. But * the C header files define this type as TimestampTz. Since the field is * potentially-null and therefore cannot be accessed directly from C code, @@ -85,18 +88,21 @@ CATALOG(pg_resqueue,6026) BKI_SHARED_RELATION { - NameData rsqname; /* name of resource queue */ - Oid rsq_parent; /* oid of resource queue */ - int4 rsq_active_stats_cluster; /* active statement count limit in cluster */ - text rsq_memory_limit_cluster; /* memory limit in cluster */ - text rsq_core_limit_cluster; /* core limit in cluster */ - float4 rsq_resource_upper_factor; /* resource upper limit in cluster */ - text rsq_allocation_policy; /* query resource allocation policy */ - text rsq_vseg_resource_quota; /* vsegment resource quota */ - int4 rsq_vseg_upper_limit; /* vsegment size upper limit */ - timestamptz rsq_creation_time; /* when the queue is created */ - timestamptz rsq_update_time; /* when the queue is updated ( create or alter ) */ - text rsq_status; /* the status of resource queue. */ + NameData rsqname; /* name of resource queue */ + Oid parentoid; /* oid of resource queue */ + int4 activestats; /* active statement count limit */ + text memorylimit; /* memory limit in cluster */ + text corelimit; /* core limit in cluster */ + float4 resovercommit; /* resource upper limit in cluster */ + text allocpolicy; /* query resource allocation policy */ + text vsegresourcequota; /* vsegment resource quota */ + int4 nvsegupperlimit; /* vsegment size upper limit */ + int4 nvseglowerlimit; /* vsegment size lower limit */ + float4 nvsegupperlimitperseg; /* vsegment size upper limit per segment */ + float4 nvseglowerlimitperseg; /* vsegment size lower limit per segment */ + timestamptz creationtime; /* when the queue is created */ + timestamptz updatetime; /* when the queue is updated ( create or alter ) */ + text status; /* the status of resource queue */ } FormData_pg_resqueue; #undef timestamptz @@ -114,268 +120,95 @@ typedef FormData_pg_resqueue *Form_pg_resqueue; * compiler constants for pg_resqueue * ---------------- */ -#define Natts_pg_resqueue 12 -#define Anum_pg_resqueue_rsqname 1 -#define Anum_pg_resqueue_rsq_parent 2 -#define Anum_pg_resqueue_rsq_active_stats_cluster 3 -#define Anum_pg_resqueue_rsq_memory_limit_cluster 4 -#define Anum_pg_resqueue_rsq_core_limit_cluster 5 -#define Anum_pg_resqueue_rsq_resource_upper_factor 6 -#define Anum_pg_resqueue_rsq_allocation_policy 7 -#define Anum_pg_resqueue_rsq_vseg_resource_quota 8 -#define Anum_pg_resqueue_rsq_vseg_upper_limit 9 -#define Anum_pg_resqueue_rsq_creation_time 10 -#define Anum_pg_resqueue_rsq_update_time 11 -#define Anum_pg_resqueue_rsq_status 12 - -/* TIDYCAT_END_CODEGEN */ - - -/* Create initial default resource queue */ -DATA(insert OID = 9800 ( "pg_root" 0 "-1" "100%" "100%" 2 "even" _null_ _null_ _null_ _null_ "branch")); -DATA(insert OID = 6055 ( "pg_default" 9800 "100" "50%" "50%" 2 "even" "mem:128mb" -1 _null_ _null_ _null_)); - -#define DEFAULT_RESQUEUE_ACTIVESTATS "100" -#define DEFAULT_RESQUEUE_UPPERFACTOR "2" -#define DEFAULT_RESQUEUE_VSEG_UPPER_LIMIT "-1" -#define DEFAULT_RESQUEUE_VSEG_UPPER_LIMIT_N -1 - -#define MINIMUM_RESQUEUE_UPPER_FACTOR_LIMIT "1" -#define MINIMUM_RESQUEUE_UPPER_FACTOR_LIMIT_N 1.0 - -#define DEFAULT_RESQUEUE_POLICY "even" -#define DEFAULT_RESQUEUE_SEG_QUOTA "mem:128mb" - -#define PG_RESQUEUE_COL_OID "oid" -#define PG_RESQUEUE_COL_RSQNAME "rsqname" -#define PG_RESQUEUE_COL_PARENT "rsq_parent" -#define PG_RESQUEUE_COL_ACTIVE_STATS_CLUSTER "rsq_active_stats_cluster" -#define PG_RESQUEUE_COL_MEMORY_LIMIT_CLUSTER "rsq_memory_limit_cluster" -#define PG_RESQUEUE_COL_CORE_LIMIT_CLUSTER "rsq_core_limit_cluster" -#define PG_RESQUEUE_COL_RESOURCE_UPPER_FACTOR "rsq_resource_upper_factor" -#define PG_RESQUEUE_COL_ALLOCATION_POLICY "rsq_allocation_policy" -#define PG_RESQUEUE_COL_VSEG_RESOURCE_QUOTA "rsq_vseg_resource_quota" -#define PG_RESQUEUE_COL_VSEG_UPPER_LIMIT "rsq_vseg_upper_limit" -#define PG_RESQUEUE_COL_CREATION_TIME "rsq_creation_time" -#define PG_RESQUEUE_COL_UPDATE_TIME "rsq_update_time" -#define PG_RESQUEUE_COL_STATUS "rsq_status" - -#define ROOTRESQUEUE_OID 9800 -#define DEFAULTRESQUEUE_OID 6055 - -/* - The flavors of resource types: - - required: user must specify this type during CREATE. Every queue - must always have this type entry in pg_resqueuecapability. It is - not required to have a default value. It may or may not have an - offvalue, depending on the "has disable" setting. - - optional (ie not required): user does not have to specify this type. - If hasdefault is false, then no entry is required for - pg_resqueuecapability. If hasdefault is true, then CREATE will add - the default entry to pg_resqueuecapability during CREATE. - - has disable: whether the resource type has an OFF switch, ie what is - the WITHOUT behavior. For a required type, if it can be disabled, - then it must have an off value. For an optional type, if it can be - disabled, there are two options: - - 1) if the optional type has a default value, then if must have an - off value. The off value can be the same as the default value. - - 2) if the optional type does not have a default value, then the - assumption is that it gets "shut off" by removing the - pg_resqueuecapability entry. The off value is ignored. Which - means if hasdefault is false, and required is false, then - hasdisable must be true (because the CREATE statement won't add - an entry for the type, so it is de facto disabled). - - */ - -/* MPP-6923: Resource Queue attribute flexibility */ - -/* TIDYCAT_BEGINDEF - - CREATE TABLE pg_resourcetype - with (camelcase=ResourceType, shared=true, relid=6059, reltype_oid=6445) - ( - resname name, -- name of resource type - restypid smallint, -- resource type id - resrequired boolean, -- if required, user must specify during CREATE - reshasdefault boolean, -- create a default entry for optional type - reshasdisable boolean, -- whether the type can be removed or shut off - resdefaultsetting text, -- default resource setting - resdisabledsetting text -- value that turns it off - - ); - - create unique index on pg_resourcetype(oid) with (indexid=6061); - create unique index on pg_resourcetype(restypid) with (indexid=6062); - create unique index on pg_resourcetype(resname) with (indexid=6063); - - TIDYCAT_ENDDEF -*/ -/* TIDYCAT_BEGIN_CODEGEN - - WARNING: DO NOT MODIFY THE FOLLOWING SECTION: - Generated by ./tidycat.pl version 34 - on Thu Aug 7 17:27:40 2014 -*/ - - -/* - TidyCat Comments for pg_resourcetype: - Table is shared, so catalog.c:IsSharedRelation is updated. - Table has an Oid column. - Table has static type (see pg_types.h). - Table has TOASTable columns, but NO TOAST table. - -*/ - -/* ---------------- - * pg_resourcetype definition. cpp turns this into - * typedef struct FormData_pg_resourcetype - * ---------------- - */ -#define ResourceTypeRelationId 6059 - -CATALOG(pg_resourcetype,6059) BKI_SHARED_RELATION -{ - NameData resname; /* name of resource type */ - int2 restypid; /* resource type id */ - bool resrequired; /* if required, user must specify during CREATE */ - bool reshasdefault; /* create a default entry for optional type */ - bool reshasdisable; /* whether the type can be removed or shut off */ - text resdefaultsetting; /* default resource setting */ - text resdisabledsetting; /* value that turns it off */ -} FormData_pg_resourcetype; - - -/* ---------------- - * Form_pg_resourcetype corresponds to a pointer to a tuple with - * the format of pg_resourcetype relation. - * ---------------- - */ -typedef FormData_pg_resourcetype *Form_pg_resourcetype; - - -/* ---------------- - * compiler constants for pg_resourcetype - * ---------------- - */ -#define Natts_pg_resourcetype 7 -#define Anum_pg_resourcetype_resname 1 -#define Anum_pg_resourcetype_restypid 2 -#define Anum_pg_resourcetype_resrequired 3 -#define Anum_pg_resourcetype_reshasdefault 4 -#define Anum_pg_resourcetype_reshasdisable 5 -#define Anum_pg_resourcetype_resdefaultsetting 6 -#define Anum_pg_resourcetype_resdisabledsetting 7 +#define Natts_pg_resqueue 15 +#define Anum_pg_resqueue_rsqname 1 +#define Anum_pg_resqueue_parentoid 2 +#define Anum_pg_resqueue_activestats 3 +#define Anum_pg_resqueue_memorylimit 4 +#define Anum_pg_resqueue_corelimit 5 +#define Anum_pg_resqueue_resovercommit 6 +#define Anum_pg_resqueue_allocpolicy 7 +#define Anum_pg_resqueue_vsegresourcequota 8 +#define Anum_pg_resqueue_nvsegupperlimit 9 +#define Anum_pg_resqueue_nvseglowerlimit 10 +#define Anum_pg_resqueue_nvsegupperlimitperseg 11 +#define Anum_pg_resqueue_nvseglowerlimitperseg 12 +#define Anum_pg_resqueue_creationtime 13 +#define Anum_pg_resqueue_updatetime 14 +#define Anum_pg_resqueue_status 15 /* TIDYCAT_END_CODEGEN */ -/* Create entry in pg_resourcetype for PRIORITY */ -DATA(insert OID = 6454 ( active_statements 1 f t f -1 -1 )); -DATA(insert OID = 6455 ( parent 2 t f f -1 -1 )); -DATA(insert OID = 6456 ( memory_limit_cluster 3 t f f -1 -1)); -DATA(insert OID = 6457 ( core_limit_cluster 4 t f f -1 -1 )); -DATA(insert OID = 6458 ( vsegment_resource_quota 5 f t f -1 -1 )); -DATA(insert OID = 6459 ( resource_upper_factor 6 f t f -1 -1 )); -/* - The first four entries of pg_resourcetype are special mappings for - the original pg_resqueue columns. The following table shows the - correspondence between the original grammar, the pg_resqueue column - name, and the WITH clause defnames. - - grammar colname orig_defname new_defname - ------------------ ----------------- --------------- ----------------- - "ACTIVE THRESHOLD" rsqcountlimit activelimit active_statements - "COST THRESHOLD" rsqcostlimit costlimit max_cost - "IGNORE THRESHOLD" rsqignorecostlimit ignorecostlimit min_cost - "OVERCOMMIT" rsqovercommit overcommit cost_overcommit -*/ - -/* Note: the restypid is used by pg_dumpall.c to build CREATE statements */ -#define PG_RESRCTYPE_ACTIVE_STATEMENTS 1 /* rsqcountlimit: count */ -#define PG_RESRCTYPE_MAX_COST 2 /* rsqcostlimit: max_cost */ -#define PG_RESRCTYPE_MIN_COST 3 /* rsqignorecostlimit: min_cost */ -#define PG_RESRCTYPE_COST_OVERCOMMIT 4 /* rsqovercommit: cost_overcommit*/ -/* start of "pg_resourcetype" entries... */ -#define PG_RESRCTYPE_PRIORITY 5 /* backoff.c: priority queue */ -#define PG_RESRCTYPE_MEMORY_LIMIT 6 /* memquota.c: memory quota */ - -/* TIDYCAT_BEGINDEF - - CREATE TABLE pg_resqueuecapability - with (camelcase=ResQueueCapability, shared=true, relid=6060, reltype_oid=6446) - ( - resqueueid oid, -- OID of the queue with this capability - restypid smallint, -- resource type id (key to pg_resourcetype) - ressetting text -- resource setting (opaque type) - ); - - create unique index on pg_resqueuecapability(oid) with (indexid=6064); - create index on pg_resqueuecapability(resqueueid) with (indexid=6065); - create index on pg_resqueuecapability(restypid) with (indexid=6066); - - alter table pg_resqueuecapability add fk resqueueid on pg_resqueue(oid); - alter table pg_resqueuecapability add fk restypid on pg_resourcetype(restypid); - - TIDYCAT_ENDDEF -*/ -/* TIDYCAT_BEGIN_CODEGEN - - WARNING: DO NOT MODIFY THE FOLLOWING SECTION: - Generated by ./tidycat.pl version 34 - on Thu Aug 7 17:27:40 2014 -*/ +/* Create initial default resource queue */ +DATA(insert OID = 9800 ( "pg_root" 0 "-1" "100%" "100%" 2 "even" _null_ 0 0 0 1 _null_ _null_ "branch")); +DATA(insert OID = 6055 ( "pg_default" 9800 "100" "50%" "50%" 2 "even" "mem:128mb" 0 0 0 1 _null_ _null_ _null_)); /* - TidyCat Comments for pg_resqueuecapability: - Table is shared, so catalog.c:IsSharedRelation is updated. - Table has an Oid column. - Table has static type (see pg_types.h). - Table has TOASTable columns, but NO TOAST table. - -*/ - -/* ---------------- - * pg_resqueuecapability definition. cpp turns this into - * typedef struct FormData_pg_resqueuecapability - * ---------------- - */ -#define ResQueueCapabilityRelationId 6060 - -CATALOG(pg_resqueuecapability,6060) BKI_SHARED_RELATION -{ - Oid resqueueid; /* OID of the queue with this capability */ - int2 restypid; /* resource type id (key to pg_resourcetype) */ - text ressetting; /* resource setting (opaque type) */ -} FormData_pg_resqueuecapability; - - -/* ---------------- - * Form_pg_resqueuecapability corresponds to a pointer to a tuple with - * the format of pg_resqueuecapability relation. - * ---------------- - */ -typedef FormData_pg_resqueuecapability *Form_pg_resqueuecapability; - - -/* ---------------- - * compiler constants for pg_resqueuecapability - * ---------------- + * The possible resource allocation policies. */ -#define Natts_pg_resqueuecapability 3 -#define Anum_pg_resqueuecapability_resqueueid 1 -#define Anum_pg_resqueuecapability_restypid 2 -#define Anum_pg_resqueuecapability_ressetting 3 - +enum RESOURCE_QUEUE_ALLOCATION_POLICY_INDEX { + RSQ_ALLOCATION_POLICY_EVEN = 0, + + RSQ_ALLOCATION_POLICY_COUNT +}; + +#define DEFAULT_RESQUEUE_ACTIVESTATS "100" +#define DEFAULT_RESQUEUE_OVERCOMMIT "2" +#define DEFAULT_RESQUEUE_NVSEG_UPPER_LIMIT "0" +#define DEFAULT_RESQUEUE_NVSEG_LOWER_LIMIT "0" +#define DEFAULT_RESQUEUE_NVSEG_UPPER_PERSEG_LIMIT "0" +#define DEFAULT_RESQUEUE_NVSEG_LOWER_PERSEG_LIMIT "1" + +#define DEFAULT_RESQUEUE_ALLOCPOLICY "even" +#define DEFAULT_RESQUEUE_VSEGRESOURCEQUOTA "mem:128mb" + +#define DEFAULT_RESQUEUE_ACTIVESTATS_N 100 +#define DEFAULT_RESQUEUE_OVERCOMMIT_N 2.0 +#define DEFAULT_RESQUEUE_NVSEG_UPPER_LIMIT_N 0 +#define DEFAULT_RESQUEUE_NVSEG_LOWER_LIMIT_N 0 +#define DEFAULT_RESQUEUE_NVSEG_UPPER_PERSEG_LIMIT_N 0.0 +#define DEFAULT_RESQUEUE_NVSEG_LOWER_PERSEG_LIMIT_N 1.0 + +#define DEFAULT_RESQUEUE_ALLOCPOLICY_N RSQ_ALLOCATION_POLICY_EVEN +#define DEFAULT_RESQUEUE_VSEGRESOURCEQUOTA_N 128 + +#define MINIMUM_RESQUEUE_OVERCOMMIT "1" +#define MINIMUM_RESQUEUE_NVSEG_UPPER_LIMIT "0" +#define MINIMUM_RESQUEUE_NVSEG_LOWER_LIMIT "0" +#define MINIMUM_RESQUEUE_NVSEG_UPPER_PERSEG_LIMIT "0" +#define MINIMUM_RESQUEUE_NVSEG_LOWER_PERSEG_LIMIT "0" + +#define MINIMUM_RESQUEUE_OVERCOMMIT_N 1.0 +#define MINIMUM_RESQUEUE_NVSEG_UPPER_LIMIT_N 0 +#define MINIMUM_RESQUEUE_NVSEG_LOWER_LIMIT_N 0 +#define MINIMUM_RESQUEUE_NVSEG_UPPER_PERSEG_LIMIT_N 0.0 +#define MINIMUM_RESQUEUE_NVSEG_LOWER_PERSEG_LIMIT_N 0.0 + +#define PG_RESQUEUE_COL_OID "oid" +#define PG_RESQUEUE_COL_RSQNAME "rsqname" +#define PG_RESQUEUE_COL_PARENTOID "parentoid" +#define PG_RESQUEUE_COL_ACTIVESTATS "activestats" +#define PG_RESQUEUE_COL_MEMORYLIMIT "memorylimit" +#define PG_RESQUEUE_COL_CORELIMIT "corelimit" +#define PG_RESQUEUE_COL_RESOVERCOMMIT "resovercommit" +#define PG_RESQUEUE_COL_ALLOCPOLICY "allocpolicy" +#define PG_RESQUEUE_COL_VSEGRESOURCEQUOTA "vsegresourcequota" +#define PG_RESQUEUE_COL_NVSEGUPPERLIMIT "nvsegupperlimit" +#define PG_RESQUEUE_COL_NVSEGLOWERLIMIT "nvsegupperlimit" +#define PG_RESQUEUE_COL_NVSEGUPPERLIMITPERSEG "nvsegupperlimitperseg" +#define PG_RESQUEUE_COL_NVSEGLOWERLIMITPERSEG "nvseglowerlimitperseg" +#define PG_RESQUEUE_COL_CREATIONTIME "creationtime" +#define PG_RESQUEUE_COL_UPDATETIME "updatetime" +#define PG_RESQUEUE_COL_STATUS "status" -/* TIDYCAT_END_CODEGEN */ +#define ROOTRESQUEUE_OID 9800 +#define DEFAULTRESQUEUE_OID 6055 +#define RESOURCE_QUEUE_DEFAULT_QUEUE_NAME "pg_default" +#define RESOURCE_QUEUE_ROOT_QUEUE_NAME "pg_root" +#define RESOURCE_QUEUE_SEG_RES_QUOTA_MEM "mem:" +#define RESOURCE_QUEUE_SEG_RES_QUOTA_CORE "core:" /* Reserved. */ #endif /* PG_RESQUEUE_H */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a413a426/src/include/catalog/pg_type.h ---------------------------------------------------------------------- diff --git a/src/include/catalog/pg_type.h b/src/include/catalog/pg_type.h index 591c991..c356f14 100644 --- a/src/include/catalog/pg_type.h +++ b/src/include/catalog/pg_type.h @@ -793,22 +793,14 @@ DATA(insert OID = 7077 ( pg_remote_credentials PGNSP PGUID -1 f c t \054 707 /* relation id: 7076 - pg_remote_credentials 20140205 */ DATA(insert OID = 7080 (pg_toast_7076 TOASTNSP PGUID -1 f c t \054 7078 0 record_in record_out record_recv record_send - d x f 0 -1 0 _null_ _null_)); #define PG_REMOTE_CREDENTIALS_TOAST_RELTYPE_OID 7080 -/* relation id: 6026 - pg_resqueue 20140807 */ +/* relation id: 6026 - pg_resqueue 20150917 */ DATA(insert OID = 9830 ( pg_resqueue PGNSP PGUID -1 f c t \054 6026 0 record_in record_out record_recv record_send - d x f 0 -1 0 _null_ _null_ )); #define PG_RESQUEUE_RELTYPE_OID 9830 -/* relation id: 6026 - pg_resqueue 20140807 */ +/* relation id: 6026 - pg_resqueue 20150917 */ DATA(insert OID = 9822 (pg_toast_6026 TOASTNSP PGUID -1 f c t \054 9820 0 record_in record_out record_recv record_send - d x f 0 -1 0 _null_ _null_)); #define PG_RESQUEUE_TOAST_RELTYPE_OID 9822 -/* relation id: 6059 - pg_resourcetype 20140807 */ -DATA(insert OID = 6445 ( pg_resourcetype PGNSP PGUID -1 f c t \054 6059 0 record_in record_out record_recv record_send - d x f 0 -1 0 _null_ _null_ )); -#define PG_RESOURCETYPE_RELTYPE_OID 6445 - -/* relation id: 6060 - pg_resqueuecapability 20140807 */ -DATA(insert OID = 6446 ( pg_resqueuecapability PGNSP PGUID -1 f c t \054 6060 0 record_in record_out record_recv record_send - d x f 0 -1 0 _null_ _null_ )); -#define PG_RESQUEUECAPABILITY_RELTYPE_OID 6446 - /* TIDYCAT_END_CODEGEN */ DATA(insert OID = 6989 (gp_persistent_relfile_node PGNSP PGUID -1 f c t \054 5089 0 record_in record_out record_recv record_send - d x f 0 -1 0 _null_ _null_)); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a413a426/src/include/catalog/tidycat.pl ---------------------------------------------------------------------- diff --git a/src/include/catalog/tidycat.pl b/src/include/catalog/tidycat.pl index cd9deca..d613609 100755 --- a/src/include/catalog/tidycat.pl +++ b/src/include/catalog/tidycat.pl @@ -447,7 +447,6 @@ my %dynamic_reltype_h = 5011 => "pg_partition_rule", 1136 => "pg_pltemplate", 1255 => "pg_proc", # special bootstrap case - #6026 => "pg_resqueue", 2618 => "pg_rewrite", 1214 => "pg_shdepend", 2396 => "pg_shdescription", @@ -490,8 +489,6 @@ my %toast_tab_exception_h = "pg_index" => 1, "pg_partition_rule" => 1, "pg_pltemplate" => 1, - "pg_resqueuecapability" => 1, - "pg_resourcetype" => 1, "pg_stat_last_operation" => 1, "pg_stat_last_shoperation" => 1, "pg_tablespace" => 1, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a413a426/src/include/catalog/toasting.h ---------------------------------------------------------------------- diff --git a/src/include/catalog/toasting.h b/src/include/catalog/toasting.h index 0806928..401defb 100644 --- a/src/include/catalog/toasting.h +++ b/src/include/catalog/toasting.h @@ -101,7 +101,7 @@ DECLARE_TOAST(pg_remote_credentials, 7078, 7079); #define PgRemoteCredentialsToastTable 7078 #define PgRemoteCredentialsToastIndex 7079 -/* relation id: 6026 - pg_resqueue 20140807 */ +/* relation id: 6026 - pg_resqueue 20150917 */ DECLARE_TOAST(pg_resqueue, 9820, 9821); #define PgResQueueToastTable 9820 #define PgResQueueToastIndex 9821 http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a413a426/src/include/cdb/cdbvars.h ---------------------------------------------------------------------- diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h index dcc7bdc..64d8b74 100644 --- a/src/include/cdb/cdbvars.h +++ b/src/include/cdb/cdbvars.h @@ -1155,13 +1155,12 @@ extern char *dfs_url; extern char *master_directory; extern char *seg_directory; -extern bool rm_domain_comm_enable; /* HAWQ 2.0 resource manager GUCs */ -extern int rm_master_addr_domain_port; -extern int rm_master_addr_port; -extern int rm_seg_addr_port; +extern int rm_master_domain_port; +extern int rm_master_port; +extern int rm_segment_port; -extern char *rm_grm_server_type; +extern char *rm_global_rm_type; extern char *rm_seg_memory_use; extern double rm_seg_core_use; @@ -1172,14 +1171,19 @@ extern char *rm_grm_yarn_queue; extern char *rm_grm_yarn_app_name; extern int rm_grm_breath_return_percentage; -extern int rm_query_vseg_num_limit; -extern int rm_query_vseg_num_per_seg_limit; -extern int rm_slice_num_per_seg_limit; +extern int rm_nvseg_perquery_limit; +extern int rm_nvseg_perquery_perseg_limit; +extern int rm_nslice_perseg_limit; extern int rm_seg_container_default_waterlevel; -extern int rm_resource_noaction_timeout; + +extern int rm_session_lease_timeout; +extern bool rm_session_lease_heartbeat_enable; + extern int rm_query_resource_noresource_timeout; extern int rm_resource_timeout; extern int rm_resource_heartbeat_interval; +extern int rm_tolerate_nseg_limit; +extern int rm_nvseg_variance_among_seg_limit; extern char *rm_resourcepool_test_filename; extern bool rm_force_fifo_queue; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a413a426/src/test/regress/data/upgrade20/upg2_catupgrade_20.sql.in ---------------------------------------------------------------------- diff --git a/src/test/regress/data/upgrade20/upg2_catupgrade_20.sql.in b/src/test/regress/data/upgrade20/upg2_catupgrade_20.sql.in index bd8114d..74d5ffe 100644 --- a/src/test/regress/data/upgrade20/upg2_catupgrade_20.sql.in +++ b/src/test/regress/data/upgrade20/upg2_catupgrade_20.sql.in @@ -106,7 +106,7 @@ UPDATE pg_resourcetype SET resname='active_statements', resrequired=false, resha UPDATE pg_resourcetype SET resname='parent', resrequired=true, reshasdefault=false, reshasdisable=false, resdefaultsetting=-1, resdisabledsetting=-1 WHERE restypid=2; UPDATE pg_resourcetype SET resname='memory_limit_cluster', resrequired=true, reshasdefault=false, reshasdisable=false, resdefaultsetting=-1, resdisabledsetting=-1 WHERE restypid=3; UPDATE pg_resourcetype SET resname='core_limit_cluster', resrequired=true, reshasdefault=false, reshasdisable=false, resdefaultsetting=-1, resdisabledsetting=-1 WHERE restypid=4; -UPDATE pg_resourcetype SET resname='segment_resource_quota', resrequired=false, reshasdefault=true, reshasdisable=false, resdefaultsetting=-1, resdisabledsetting=-1 WHERE restypid=5; +UPDATE pg_resourcetype SET resname='vseg_resource_quota', resrequired=false, reshasdefault=true, reshasdisable=false, resdefaultsetting=-1, resdisabledsetting=-1 WHERE restypid=5; UPDATE pg_resourcetype SET resname='resource_upper_factor', resrequired=false, reshasdefault=true, reshasdisable=false, resdefaultsetting=-1, resdisabledsetting=-1 WHERE restypid=6; -- change proc pg_stat_get_activity http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a413a426/src/test/regress/expected/parquet_compression.out ---------------------------------------------------------------------- diff --git a/src/test/regress/expected/parquet_compression.out b/src/test/regress/expected/parquet_compression.out index dd5370f..bcb7cd8 100644 --- a/src/test/regress/expected/parquet_compression.out +++ b/src/test/regress/expected/parquet_compression.out @@ -26,7 +26,7 @@ drop table parquet_snappy_part_unc; ERROR: table "parquet_snappy_part_unc" does not exist drop table parquet_gzip_2; ERROR: table "parquet_gzip_2" does not exist -alter resource queue pg_default with ( segment_resource_quota='mem:2gb'); +alter resource queue pg_default with ( vseg_resource_quota='mem:2gb'); --set statement_mem='1999MB'; --end_ignore --Datatypes covered: text,bytea,varchar,bit varying @@ -145,4 +145,4 @@ Select count(*) from parquet_gzip_2; 1 (1 row) -alter resource queue pg_default with ( vsegment_resource_quota='mem:128mb'); +alter resource queue pg_default with ( vseg_resource_quota='mem:128mb'); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a413a426/src/test/regress/sql/parquet_compression.sql ---------------------------------------------------------------------- diff --git a/src/test/regress/sql/parquet_compression.sql b/src/test/regress/sql/parquet_compression.sql index 48170cc..92a6f5f 100644 --- a/src/test/regress/sql/parquet_compression.sql +++ b/src/test/regress/sql/parquet_compression.sql @@ -19,7 +19,7 @@ drop table parquet_snappy_part; drop table parquet_snappy_part_unc; drop table parquet_gzip_2; -alter resource queue pg_default with ( vsegment_resource_quota='mem:2gb'); +alter resource queue pg_default with ( vseg_resource_quota='mem:2gb'); --set statement_mem='1999MB'; --end_ignore @@ -122,4 +122,4 @@ insert into parquet_gzip_2 values(12,array_to_string(ARRAY(SELECT chr((65 + roun Select count(*) from parquet_gzip_2; -alter resource queue pg_default with ( vsegment_resource_quota='mem:128mb'); +alter resource queue pg_default with ( vseg_resource_quota='mem:128mb'); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a413a426/tools/bin/generate-greenplum-path.sh ---------------------------------------------------------------------- diff --git a/tools/bin/generate-greenplum-path.sh b/tools/bin/generate-greenplum-path.sh index e5fff54..6bf5eae 100755 --- a/tools/bin/generate-greenplum-path.sh +++ b/tools/bin/generate-greenplum-path.sh @@ -88,9 +88,13 @@ cat << EOF LIBHDFS3_CONF=\$GPHOME/etc/hdfs-client.xml EOF +# libyarn configuration file path +cat << EOF +LIBYARN_CONF=\$GPHOME/etc/yarn-client.xml +EOF + # global resource manager configuration file path cat << EOF -RESOURCEMANAGER_CONF=\$GPHOME/etc/resourcemanager.xml HAWQSITE_CONF=\$GPHOME/etc/hawq-site.xml EOF @@ -122,6 +126,9 @@ export LIBHDFS3_CONF EOF cat <