hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hubertzh...@apache.org
Subject incubator-hawq git commit: HAWQ-999. Log_error when file count is not in proportion to bucket number of hash table.
Date Tue, 16 Aug 2016 03:00:28 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 7920ad4ec -> a8da53f40


HAWQ-999. Log_error when file count is not in proportion to bucket number of hash table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/a8da53f4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/a8da53f4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/a8da53f4

Branch: refs/heads/master
Commit: a8da53f40f07b25e74640f66ef1e0123ee74c5cf
Parents: 7920ad4
Author: hzhang2 <zhanghuan929@163.com>
Authored: Tue Aug 16 10:54:48 2016 +0800
Committer: hzhang2 <zhanghuan929@163.com>
Committed: Tue Aug 16 10:54:48 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdatalocality.c | 26 +++++++++++++++++++++++---
 src/backend/utils/misc/guc.c      | 12 ++++++++++++
 src/include/utils/guc.h           |  1 +
 3 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a8da53f4/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index aaa4c8d..3223a37 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -3791,12 +3791,27 @@ run_allocation_algorithm(SplitAllocResult *result, List *virtual_segments,
Query
 		targetPolicy = GpPolicyFetch(CurrentMemoryContext, myrelid);
 		bool isRelationHash = is_relation_hash(targetPolicy);
 
+		int fileCountInRelation = list_length(rel_data->files);
+		bool FileCountBucketNumMismatch = false;
+		if (targetPolicy->bucketnum > 0) {
+		  FileCountBucketNumMismatch = fileCountInRelation %
+		    targetPolicy->bucketnum == 0 ? false : true;
+		}
+		if (FileCountBucketNumMismatch && !allow_file_count_bucket_num_mismatch) {
+		  elog(ERROR, "file count %d in catalog is not in proportion to the bucket "
+		      "number %d of hash table with oid=%u, some data may be lost, if you "
+		      "still want to continue the query by considering the table as random, set GUC "
+		      "allow_file_count_bucket_num_mismatch to on and try again.",
+		      fileCountInRelation, targetPolicy->bucketnum, myrelid);
+		}
 		/* change the virtual segment order when keep hash.
 		 * order of idMap should also be changed.
+		 * if file count of the table is not equal to or multiple of
+		 * bucket number, we should process it as random table.
 		 */
 		if (isRelationHash && context->keep_hash
 				&& assignment_context.virtual_segment_num == targetPolicy->bucketnum
-				&& !vSegOrderChanged) {
+				&& !vSegOrderChanged && !FileCountBucketNumMismatch) {
 			change_hash_virtual_segments_order(resourcePtr, rel_data,
 					&assignment_context, &idMap);
 			for (int p = 0; p < idMap.target_segment_num; p++) {
@@ -3822,8 +3837,13 @@ run_allocation_algorithm(SplitAllocResult *result, List *virtual_segments,
Query
 		uint64_t before_run_allocate_hash_or_random = gettime_microsec();
 		/*allocate hash relation*/
 		if (isRelationHash) {
-			if (context->keep_hash && assignment_context.virtual_segment_num
-						== targetPolicy->bucketnum) {
+		  /*
+		   * if file count of the table is not equal to or multiple of
+		   * bucket number, we should process it as random table.
+		   */
+			if (context->keep_hash
+			    && assignment_context.virtual_segment_num== targetPolicy->bucketnum
+			    && !FileCountBucketNumMismatch) {
 				ListCell* parlc;
 				bool parentIsHashExist=false;
 				bool parentIsHash =false;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a8da53f4/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 7300fee..2cdcb62 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -450,6 +450,7 @@ char       *Debug_dtm_action_protocol_str;
 
 /* Enable check for compatibility of encoding and locale in createdb */
 bool		gp_encoding_check_locale_compatibility;
+bool  allow_file_count_bucket_num_mismatch;
 
 char	   *pgstat_temp_directory;
 
@@ -3176,6 +3177,17 @@ static struct config_bool ConfigureNamesBool[] =
 	},
 
 	{
+	    {"allow_file_count_bucket_num_mismatch", PGC_POSTMASTER, CLIENT_CONN_LOCALE,
+	          gettext_noop("allow hash table to be treated as random when file count and"
+	              " bucket number are mismatched"),
+	          NULL,
+	          GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE
+	    },
+	    &allow_file_count_bucket_num_mismatch,
+	    false, NULL, NULL
+	},
+
+	{
 		{"gp_temporary_files_filespace_repair", PGC_SUSET, DEVELOPER_OPTIONS,
                         gettext_noop("Change the filespace inconsistency to a warning"),
                         NULL,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/a8da53f4/src/include/utils/guc.h
----------------------------------------------------------------------
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index 67183af..41f7056 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -310,6 +310,7 @@ extern int hawq_rm_nvseg_for_analyze_nopart_perquery_perseg_limit;
 extern int hawq_rm_nvseg_for_analyze_part_perquery_perseg_limit;
 extern int hawq_rm_nvseg_for_analyze_nopart_perquery_limit;
 extern int hawq_rm_nvseg_for_analyze_part_perquery_limit;
+extern bool allow_file_count_bucket_num_mismatch;
 
 extern char *ConfigFileName;
 extern char *HbaFileName;


Mime
View raw message