Return-Path: X-Original-To: apmail-hawq-commits-archive@minotaur.apache.org Delivered-To: apmail-hawq-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A5FE919647 for ; Wed, 9 Mar 2016 09:12:41 +0000 (UTC) Received: (qmail 87524 invoked by uid 500); 9 Mar 2016 09:12:35 -0000 Delivered-To: apmail-hawq-commits-archive@hawq.apache.org Received: (qmail 87150 invoked by uid 500); 9 Mar 2016 09:12:35 -0000 Mailing-List: contact commits-help@hawq.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hawq.incubator.apache.org Delivered-To: mailing list commits@hawq.incubator.apache.org Received: (qmail 87084 invoked by uid 99); 9 Mar 2016 09:12:35 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2016 09:12:35 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 2123D180B84 for ; Wed, 9 Mar 2016 09:12:35 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -3.549 X-Spam-Level: X-Spam-Status: No, score=-3.549 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.329] autolearn=disabled Received: from mx2-lw-us.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id IIms3K9wT2-D for ; Wed, 9 Mar 2016 09:12:30 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx2-lw-us.apache.org (ASF Mail Server at mx2-lw-us.apache.org) with SMTP id 9A1C05FAD6 for ; Wed, 9 Mar 2016 09:12:29 +0000 (UTC) Received: (qmail 82644 invoked by uid 99); 9 Mar 2016 09:12:28 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 09 Mar 2016 09:12:28 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D3F9EDFC56; Wed, 9 Mar 2016 09:12:28 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hubertzhang@apache.org To: commits@hawq.incubator.apache.org Message-Id: <9dc566e00c314b90ab1898c2560262ef@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: incubator-hawq git commit: HAWQ-497. Refactor resource related GUC. Date: Wed, 9 Mar 2016 09:12:28 +0000 (UTC) Repository: incubator-hawq Updated Branches: refs/heads/master 2a694fc2c -> 675465d87 HAWQ-497. Refactor resource related GUC. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/675465d8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/675465d8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/675465d8 Branch: refs/heads/master Commit: 675465d87996cdb4daedc461034e6e6f53028233 Parents: 2a694fc Author: hubertzhang Authored: Wed Mar 9 17:11:36 2016 +0800 Committer: hubertzhang Committed: Wed Mar 9 17:11:36 2016 +0800 ---------------------------------------------------------------------- src/backend/cdb/cdbdatalocality.c | 4 +- src/backend/cdb/cdbvars.c | 1 - src/backend/commands/analyze.c | 6 +- src/backend/postmaster/identity.c | 47 ++++++- src/backend/tcop/postgres.c | 127 +++++++++++++++++++ src/backend/utils/misc/guc.c | 46 ++++++- src/include/cdb/cdbvars.h | 1 - src/include/postmaster/identity.h | 41 +++--- src/include/utils/guc.h | 5 + .../expected/create_table_distribution.out | 44 +++---- tools/bin/hawq_ctl | 12 +- 11 files changed, 267 insertions(+), 67 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/backend/cdb/cdbdatalocality.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c index f05e0b1..c1f829b 100644 --- a/src/backend/cdb/cdbdatalocality.c +++ b/src/backend/cdb/cdbdatalocality.c @@ -4034,7 +4034,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife, *0 Otherwise. */ if (isTableFunctionExists) { - context.tableFuncSegNum = default_segment_num; + context.tableFuncSegNum = GetUserDefinedFunctionVsegNum(); } /* set expected virtual segment number for hash table and external table*/ @@ -4125,7 +4125,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife, if (debug_fake_segmentnum){ fpsegnum = fopen("/tmp/segmentnumber", "w+"); - fprintf(fpsegnum, "Default segment num : %d.\n", default_segment_num); + fprintf(fpsegnum, "Default segment num : %d.\n", GetHashDistPartitionNum()); fprintf(fpsegnum, "\n"); fprintf(fpsegnum, "From random relation segment num : %d.\n", context.randomSegNum); fprintf(fpsegnum, "Result relation hash segment num : %d.\n", context.resultRelationHashSegNum); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/backend/cdb/cdbvars.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c index 12938c6..4db61ab 100644 --- a/src/backend/cdb/cdbvars.c +++ b/src/backend/cdb/cdbvars.c @@ -251,7 +251,6 @@ bool gp_parquet_insert_sort = true; /* The following GUCs is for HAWQ 2.o */ bool optimizer_enforce_hash_dist_policy; -int default_segment_num; int external_table_init_segment_num; int appendonly_split_write_size_mb; int split_read_size_mb; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/backend/commands/analyze.c ---------------------------------------------------------------------- diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c index b7d8668..dd5d5c0 100644 --- a/src/backend/commands/analyze.c +++ b/src/backend/commands/analyze.c @@ -823,7 +823,7 @@ static int calculate_virtual_segment_number(List* candidateRelations) { GpPolicy *targetPolicy = GpPolicyFetch(CurrentMemoryContext, rel->rd_id); if(targetPolicy == NULL){ - return GetUtilPartitionNum(); + return GetAnalyzeVSegNumLimit(); } if (targetPolicy->nattrs > 0) { isHashRelationExist = true; @@ -849,8 +849,8 @@ static int calculate_virtual_segment_number(List* candidateRelations) { } Assert(vsegNumber > 0); /*vsegNumber should be less than GetUtilPartitionNum*/ - if(vsegNumber > GetUtilPartitionNum()){ - vsegNumber = GetUtilPartitionNum(); + if(vsegNumber > GetAnalyzeVSegNumLimit()){ + vsegNumber = GetAnalyzeVSegNumLimit(); } return vsegNumber; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/backend/postmaster/identity.c ---------------------------------------------------------------------- diff --git a/src/backend/postmaster/identity.c b/src/backend/postmaster/identity.c index 7851fa8..b209ec5 100644 --- a/src/backend/postmaster/identity.c +++ b/src/backend/postmaster/identity.c @@ -35,8 +35,10 @@ #include "storage/proc.h" /* MyProc */ #include "executor/execdesc.h" /* AllocateResource */ #include "cdb/cdbutil.h" /* QueryResource */ +#include "cdb/cdbvars.h" #include "optimizer/cost.h" #include "utils/guc.h" + #include "commands/defrem.h" /* defGetInt64() */ #include "resourcemanager/envswitch.h" @@ -54,6 +56,8 @@ #include "cdb/cdbvars.h" #endif +int slaveHostNumber; + typedef enum SegmentRole { SEGMENT_ROLE_INVALID, @@ -452,10 +456,20 @@ GetQEGangNum(void) return SegmentId.pid.init ? SegmentId.pid.gang_member_num : 0; } -int -GetUtilPartitionNum(void) +int GetAnalyzeVSegNumLimit(void) { - return default_segment_num; + int nvseg = hawq_rm_nvseg_for_analyze_perquery_perseg_limit * slaveHostNumber; + while(nvseg > hawq_rm_nvseg_for_analyze_perquery_limit){ + nvseg -= slaveHostNumber; + } + if(nvseg <= 0){ + nvseg = hawq_rm_nvseg_for_analyze_perquery_limit; + } + return nvseg; +} + +int GetCopyFromVSegNum(void){ + return hawq_rm_nvseg_for_copy_from_perquery; } int @@ -469,7 +483,7 @@ GetPlannerSegmentNum(void) return gp_segments_for_planner; } - return GetUtilPartitionNum(); + return GetQueryVsegNum(); } /** * Read Relation Option from statement WITH options @@ -525,19 +539,38 @@ GetRelOpt_bucket_num_fromRangeVar(const RangeVar* rel_rv, int default_val) int GetRandomDistPartitionNum(void) { - return default_segment_num; + return default_hash_table_bucket_number; } int GetHashDistPartitionNum(void) { - return default_segment_num; + return default_hash_table_bucket_number; +} + +int +GetQueryVsegNum(void) +{ + int nvseg = rm_nvseg_perquery_perseg_limit * slaveHostNumber; + while(nvseg > rm_nvseg_perquery_limit){ + nvseg -= slaveHostNumber; + } + if(nvseg <= 0){ + nvseg = rm_nvseg_perquery_limit; + } + return nvseg; } int GetExternalTablePartitionNum(void) { - return default_segment_num; + return GetQueryVsegNum(); +} + +int +GetUserDefinedFunctionVsegNum(void) +{ + return GetQueryVsegNum(); } int http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/backend/tcop/postgres.c ---------------------------------------------------------------------- diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index c42046d..f9b2d57 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -240,6 +240,7 @@ extern void CheckForQDMirroringWork(void); extern bool ResourceScheduler; static int64 master_start_time = 0; +static int64_t slavesFileTimestamp = 0; static bool has_been_moved_to_cgroup = false; /* ---------------------------------------------------------------- @@ -261,6 +262,9 @@ static bool IsTransactionStmtList(List *nodes); static void SigHupHandler(SIGNAL_ARGS); static void log_disconnections(int code, Datum arg); static bool renice_current_process(int nice_level); +static int getSlaveHostNumber(FILE *fp); +static bool CheckSlaveFile(); + /* * Change the priority of the current process to the specified level @@ -4572,7 +4576,16 @@ PostgresMain(int argc, char *argv[], const char *username) * those every time through the message loop because it'd slow down * processing of batched messages, and because we don't want to report * uncommitted updates (that confuses autovacuum). + * + * We need to check the slave file first */ + if (Gp_role == GP_ROLE_DISPATCH){ + /* set slave host number which enable to set query vseg dynamically*/ + if(!CheckSlaveFile()){ + slaveHostNumber = 1; + } + } + if (send_ready_for_query) { if (IsTransactionOrTransactionBlock()) @@ -6204,3 +6217,117 @@ SyncAgentMain(int argc, char *argv[], const char *username) return 1; /* keep compiler quiet */ } + + + +static int getSlaveHostNumber(FILE *fp) +{ + int slavecnt = 0; + bool haserror = false; + bool incomment = false; + char *buffer = NULL;; + int32_t size = -1; + int32_t cursor = -1; + while( true ) + { + int c = fgetc(fp); + if ( c == EOF ) + { + if ( feof(fp) == 0 ) + { + elog(WARNING, "Failed to read slaves file, ferror() gets %d", + ferror(fp)); + haserror = true; + slavecnt = 1; + } + + break; + } + + if ( c == '\t' || c == ' ' || c == '\r' ) + { + continue; + } + + if ( c == '\n' ) + { + if ( cursor + 1 > 0 ) + { + cursor = -1; + slavecnt++; + } + incomment = false; + } + /* '#' is treated as a start symbol of a comment string in the line. */ + else if ( c == '#' ) + { + incomment = true; + } + else if ( !incomment ) + { + cursor++; + } + } + + if ( cursor + 1 > 0 ) + { + slavecnt++; + } + + return slavecnt; +} +static bool CheckSlaveFile(){ + char *filename = NULL; + if ( filename == NULL ) + { + + char *gphome = getenv("GPHOME"); + if ( gphome == NULL ) + { + elog(WARNING, "The environment variable GPHOME is not set. " + "can not find file slaves."); + return false; + } + + filename = (char*)palloc(strlen(gphome) + sizeof("/etc/slaves")); + + sprintf(filename, "%s%s", gphome, "/etc/slaves"); + } + + elog(DEBUG3, "Postmaster reads slaves file %s.", filename); + + /* Get file stat. */ + struct stat filestat; + FILE *fp = fopen(filename, "r"); + if ( fp == NULL ) + { + pfree(filename); + elog(WARNING, "Fail to open slaves file %s. errno %d", filename, errno); + return false; + } + int fd = fileno(fp); + + int fres = fstat(fd, &filestat); + if ( fres != 0 ) + { + pfree(filename); + fclose(fp); + elog(WARNING, "Fail to get slaves file stat %s. errno %d", filename, errno); + return false; + } + + int64_t filechangetime = filestat.st_mtime; + + if ( filechangetime != slavesFileTimestamp ) + { + slaveHostNumber = getSlaveHostNumber(fp); + slavesFileTimestamp = filechangetime; + } + + + + pfree(filename); + fclose(fp); + return true; +} + http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/backend/utils/misc/guc.c ---------------------------------------------------------------------- diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index a8f3f37..0321b9f 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -733,6 +733,10 @@ bool optimizer_sample_plans; int optimizer_plan_id; int optimizer_samples_number; int optimizer_log_failure; +int default_hash_table_bucket_number; +int hawq_rm_nvseg_for_copy_from_perquery; +int hawq_rm_nvseg_for_analyze_perquery_perseg_limit; +int hawq_rm_nvseg_for_analyze_perquery_limit; double optimizer_cost_threshold; double optimizer_nestloop_factor; double locality_upper_bound; @@ -4495,15 +4499,43 @@ static struct config_int ConfigureNamesInt[] = &join_collapse_limit, 20, 1, INT_MAX, NULL, NULL }, + { - {"default_segment_num", PGC_USERSET, QUERY_TUNING_OTHER, - gettext_noop("Sets default segment number"), - NULL, - GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE + {"default_hash_table_bucket_number", PGC_USERSET, QUERY_TUNING_OTHER, + gettext_noop("Sets default hash table bucket number"), + NULL }, - &default_segment_num, - 8, 1, INT_MAX, NULL, NULL + &default_hash_table_bucket_number, + 6, 1, INT_MAX, NULL, NULL }, + + { + {"hawq_rm_nvseg_for_copy_from_perquery", PGC_USERSET, QUERY_TUNING_OTHER, + gettext_noop("Sets default virtual segment number for copy from statement"), + NULL + }, + &hawq_rm_nvseg_for_copy_from_perquery, + 6, 1, INT_MAX, NULL, NULL + }, + + { + {"hawq_rm_nvseg_for_analyze_perquery_perseg_limit", PGC_USERSET, QUERY_TUNING_OTHER, + gettext_noop("Sets default virtual segment number per query per segment limit for analyze statement"), + NULL + }, + &hawq_rm_nvseg_for_analyze_perquery_perseg_limit, + 4, 1, INT_MAX, NULL, NULL + }, + + { + {"hawq_rm_nvseg_for_analyze_perquery_limit", PGC_USERSET, QUERY_TUNING_OTHER, + gettext_noop("Sets default virtual segment number per query limit for analyze statement"), + NULL + }, + &hawq_rm_nvseg_for_analyze_perquery_limit, + 256, 1, INT_MAX, NULL, NULL + }, + { {"enforce_virtual_segment_number", PGC_USERSET, QUERY_TUNING_OTHER, gettext_noop("Sets virtual segment number manually"), @@ -6327,7 +6359,7 @@ static struct config_int ConfigureNamesInt[] = NULL }, &rm_nvseg_perquery_perseg_limit, - 8, 1, 65535, NULL, NULL + 6, 1, 65535, NULL, NULL }, { http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/include/cdb/cdbvars.h ---------------------------------------------------------------------- diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h index 6512a4e..c1227b4 100644 --- a/src/include/cdb/cdbvars.h +++ b/src/include/cdb/cdbvars.h @@ -1111,7 +1111,6 @@ extern int gp_temporary_directory_mark_error; /* HAWQ 2.0 GUCs */ extern bool optimizer_enforce_hash_dist_policy; -extern int default_segment_num; extern int appendonly_split_write_size_mb; extern int split_read_size_mb; extern int enforce_virtual_segment_number; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/include/postmaster/identity.h ---------------------------------------------------------------------- diff --git a/src/include/postmaster/identity.h b/src/include/postmaster/identity.h index 7eeabf9..814935e 100644 --- a/src/include/postmaster/identity.h +++ b/src/include/postmaster/identity.h @@ -66,30 +66,35 @@ typedef struct ProcessIdentity ProcessFunctionList function; } ProcessIdentity; +extern int slaveHostNumber; + extern void SetSegmentIdentity(const char *name); extern bool IsOnMaster(void); extern const char *SerializeProcessIdentity(struct ProcessIdentity *id, int *msg_len); extern bool SetupProcessIdentity(const char *str); extern void SetupDispatcherIdentity(int segmentNum); -extern bool AmIMaster(void); -extern bool AmIStandby(void); -extern bool AmISegment(void); - -extern bool InitQDContext(struct ProcessIdentity *id); - -extern bool InitQEContext(struct ProcessIdentity *id, const char *str); -extern int GetQEIndex(void); -extern int GetQEGangNum(void); -extern int GetPlannerSegmentNum(void); -extern int GetUtilPartitionNum(void); -extern int GetRelOpt_bucket_num_fromOptions(List *options, int default_val); -extern int GetRelOpt_bucket_num_fromRel(Relation relation, int default_val); -extern int GetRelOpt_bucket_num_fromRangeVar(const RangeVar* rel_rv, int default_val); -extern int GetRandomDistPartitionNum(void); -extern int GetHashDistPartitionNum(void); -extern int GetExternalTablePartitionNum(void); -extern int GetAllWorkerHostNum(void); +extern bool AmIMaster(void); +extern bool AmIStandby(void); +extern bool AmISegment(void); + +extern bool InitQDContext(struct ProcessIdentity *id); + +extern bool InitQEContext(struct ProcessIdentity *id, const char *str); +extern int GetQEIndex(void); +extern int GetQEGangNum(void); +extern int GetPlannerSegmentNum(void); +extern int GetQueryVsegNum(void); +extern int GetUserDefinedFunctionVsegNum(void); +extern int GetAnalyzeVSegNum(void); +extern int GetCopyFromVSegNum(void); +extern int GetRelOpt_bucket_num_fromOptions(List *options, int default_val); +extern int GetRelOpt_bucket_num_fromRel(Relation relation, int default_val); +extern int GetRelOpt_bucket_num_fromRangeVar(const RangeVar* rel_rv, int default_val); +extern int GetRandomDistPartitionNum(void); +extern int GetHashDistPartitionNum(void); +extern int GetExternalTablePartitionNum(void); +extern int GetAllWorkerHostNum(void); extern SegmentFunctionList *GetSegmentFunctionList(void); extern ProcessFunctionList *GetProcessFunctionList(void); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/include/utils/guc.h ---------------------------------------------------------------------- diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index befeaf6..06fc75a 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -308,6 +308,11 @@ extern int Debug_dtm_action_protocol; extern int Debug_dtm_action_delay_ms; extern int Debug_dtm_action_segment; +extern int default_hash_table_bucket_number; +extern int hawq_rm_nvseg_for_copy_from_perquery; +extern int hawq_rm_nvseg_for_analyze_perquery_perseg_limit; +extern int hawq_rm_nvseg_for_analyze_perquery_limit; + extern char *ConfigFileName; extern char *HbaFileName; extern char *IdentFileName; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/src/test/regress/expected/create_table_distribution.out ---------------------------------------------------------------------- diff --git a/src/test/regress/expected/create_table_distribution.out b/src/test/regress/expected/create_table_distribution.out index e68a519..857b103 100644 --- a/src/test/regress/expected/create_table_distribution.out +++ b/src/test/regress/expected/create_table_distribution.out @@ -9,13 +9,13 @@ NOTICE: Table has parent, setting distribution columns to match parent table SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_1'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_1_w'); @@ -33,19 +33,19 @@ CREATE TABLE t1_1_6(c2 int) INHERITS (t1) WITH (bucketnum = 7) DISTRIBUTED RANDO SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_1_1'); bucketnum | attrnums -----------+---------- - 8 | {1} + 6 | {1} (1 row) SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_1_2'); bucketnum | attrnums -----------+---------- - 8 | {2} + 6 | {2} (1 row) SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_1_3'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_1_4'); @@ -73,7 +73,7 @@ NOTICE: Table doesn't have 'distributed by' clause, defaulting to distribution SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_2'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_2_w'); @@ -86,14 +86,14 @@ CREATE TABLE t1_2_1(LIKE t1) DISTRIBUTED BY (c1); SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_2_1'); bucketnum | attrnums -----------+---------- - 8 | {1} + 6 | {1} (1 row) CREATE TABLE t1_2_2(LIKE t1) DISTRIBUTED RANDOMLY; SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_2_2'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) CREATE TABLE t1_2_3(LIKE t1) WITH (bucketnum = 4) DISTRIBUTED BY (c1); @@ -114,7 +114,7 @@ CREATE TABLE t1_3 AS (SELECT * FROM t1); SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_3'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) CREATE TABLE t1_3_w WITH (bucketnum = 4) AS (SELECT * FROM t1); @@ -129,13 +129,13 @@ CREATE TABLE t1_3_2 AS (SELECT * FROM t1) DISTRIBUTED RANDOMLY; SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_3_1'); bucketnum | attrnums -----------+---------- - 8 | {1} + 6 | {1} (1 row) SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't1_3_2'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) CREATE TABLE t1_3_3 WITH (bucketnum = 6) AS (SELECT * FROM t1) DISTRIBUTED BY (c1); @@ -157,7 +157,7 @@ CREATE TABLE t2(c1 int) DISTRIBUTED BY (c1); SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2'); bucketnum | attrnums -----------+---------- - 8 | {1} + 6 | {1} (1 row) CREATE TABLE t2_1(c2 int) INHERITS (t2); @@ -165,7 +165,7 @@ NOTICE: Table has parent, setting distribution columns to match parent table SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_1'); bucketnum | attrnums -----------+---------- - 8 | {1} + 6 | {1} (1 row) CREATE TABLE t2_1_w(c2 int) INHERITS (t2) WITH (bucketnum = 3); @@ -182,19 +182,19 @@ CREATE TABLE t2_1_3(c2 int) INHERITS (t2) DISTRIBUTED RANDOMLY; SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_1_1'); bucketnum | attrnums -----------+---------- - 8 | {1} + 6 | {1} (1 row) SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_1_2'); bucketnum | attrnums -----------+---------- - 8 | {2} + 6 | {2} (1 row) SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_1_3'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) CREATE TABLE t2_1_4(c2 int) INHERITS (t2) WITH (bucketnum = 3) DISTRIBUTED BY (c1); @@ -223,7 +223,7 @@ NOTICE: Table doesn't have 'distributed by' clause, defaulting to distribution SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_2'); bucketnum | attrnums -----------+---------- - 8 | {1} + 6 | {1} (1 row) CREATE TABLE t2_2_w(LIKE t2) WITH (bucketnum = 4); @@ -238,14 +238,14 @@ CREATE TABLE t2_2_1(LIKE t2) DISTRIBUTED BY (c1); SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_2_1'); bucketnum | attrnums -----------+---------- - 8 | {1} + 6 | {1} (1 row) CREATE TABLE t2_2_2(LIKE t2) DISTRIBUTED RANDOMLY; SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_2_2'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) CREATE TABLE t2_2_3(LIKE t2) WITH (bucketnum = 5) DISTRIBUTED BY (c1); @@ -266,7 +266,7 @@ CREATE TABLE t2_3 AS (SELECT * FROM t2); SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_3'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) CREATE TABLE t2_3_w WITH (bucketnum = 4) AS (SELECT * FROM t2); @@ -281,14 +281,14 @@ CREATE TABLE t2_3_1 AS (SELECT * FROM t2) DISTRIBUTED BY (c1); SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_3_1'); bucketnum | attrnums -----------+---------- - 8 | {1} + 6 | {1} (1 row) CREATE TABLE t2_3_2 AS (SELECT * FROM t2) DISTRIBUTED RANDOMLY; SELECT bucketnum, attrnums FROM gp_distribution_policy WHERE localoid = (SELECT oid FROM pg_class WHERE relname = 't2_3_2'); bucketnum | attrnums -----------+---------- - 8 | + 6 | (1 row) CREATE TABLE t2_3_3 WITH (bucketnum = 5) AS (SELECT * FROM t2) DISTRIBUTED BY (c1); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/675465d8/tools/bin/hawq_ctl ---------------------------------------------------------------------- diff --git a/tools/bin/hawq_ctl b/tools/bin/hawq_ctl index 0f89596..89c6329 100755 --- a/tools/bin/hawq_ctl +++ b/tools/bin/hawq_ctl @@ -180,11 +180,11 @@ class HawqInit: return result def set_total_vsegment_num(self): - cmd = "%s; hawq config -c default_segment_num -v %s --skipvalidation -q > /dev/null" % \ + cmd = "%s; hawq config -c default_hash_table_bucket_number -v %s --skipvalidation -q > /dev/null" % \ (source_hawq_env, self.total_vseg_num) result = local_ssh(cmd, logger) if result != 0: - logger.warn("Set default_segment_num failed") + logger.warn("Set default_hash_table_bucket_number failed") return result def set_vsegment_num_per_node(self): @@ -332,7 +332,7 @@ class HawqInit: def _init_cluster(self): logger.info("%s segment hosts defined" % self.hosts_count_number) - logger.info("Set default_segment_num as: %s" % self.total_vseg_num) + logger.info("Set default_hash_table_bucket_number as: %s" % self.total_vseg_num) check_return_code(self.set_total_vsegment_num()) check_return_code(self.set_vsegment_num_per_node()) @@ -375,7 +375,7 @@ class HawqInit: if self.node_type == "master": self.check_hdfs_path() logger.info("%s segment hosts defined" % self.hosts_count_number) - logger.info("Set default_segment_num as: %s" % self.total_vseg_num) + logger.info("Set default_hash_table_bucket_number as: %s" % self.total_vseg_num) check_return_code(self.set_total_vsegment_num()) check_return_code(self.set_vsegment_num_per_node()) logger.info("Start to init master") @@ -1200,12 +1200,12 @@ def create_parser(): parser.add_option("--vsegNumber", type="int", dest="virtual_seg_num", - default=8, + default=6, help="Sets maximum number of virtual segments per node") parser.add_option("--vsegment-number", type="int", dest="virtual_seg_num", - default=8, + default=6, help="Sets maximum number of virtual segments per node") parser.add_option("--locale", dest="hawq_locale",