hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hubertzh...@apache.org
Subject incubator-hawq git commit: HAWQ-297. Double check hdfs and split length equals to metadata logic length.
Date Thu, 31 Dec 2015 02:06:11 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 685da7a54 -> 5e16bd49e


HAWQ-297. Double check hdfs and split length equals to metadata logic length.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/5e16bd49
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/5e16bd49
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/5e16bd49

Branch: refs/heads/master
Commit: 5e16bd49e476fdb8db61b256e32c4724a5bb4b26
Parents: 685da7a
Author: hubertzhang <hzhang@pivotal.io>
Authored: Wed Dec 30 17:59:39 2015 +0800
Committer: hubertzhang <hzhang@pivotal.io>
Committed: Thu Dec 31 09:52:39 2015 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdatalocality.c | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/5e16bd49/src/backend/cdb/cdbdatalocality.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c
index 5951ba7..3cc4e74 100644
--- a/src/backend/cdb/cdbdatalocality.c
+++ b/src/backend/cdb/cdbdatalocality.c
@@ -307,6 +307,8 @@ typedef struct split_to_segment_mapping_context {
 	int64 total_size; /* total data size for all relations */
 	int64 total_split_count;
 	int64 total_file_count;
+
+	int64 total_metadata_logic_len;
 } split_to_segment_mapping_context;
 
 typedef struct vseg_list{
@@ -343,6 +345,8 @@ static List *get_virtual_segments(QueryResource *resource);
 static List *run_allocation_algorithm(SplitAllocResult *result, List *virtual_segments, QueryResource
** resourcePtr,
 		split_to_segment_mapping_context *context);
 
+static void double_check_hdfs_metadata_logic_length(BlockLocation * locations,int block_num,int64
logic_len);
+
 static void AOGetSegFileDataLocation(Relation relation,
 		AppendOnlyEntry *aoEntry, Snapshot metadataSnapshot,
 		split_to_segment_mapping_context *context, int64 splitsize,
@@ -537,7 +541,7 @@ static void init_datalocality_context(split_to_segment_mapping_context
*context)
 	context->total_size = 0;
 	context->total_split_count = 0;
 	context->total_file_count = 0;
-
+	context->total_metadata_logic_len = 0;
 	return;
 }
 
@@ -1045,6 +1049,20 @@ update_data_dist_stat(split_to_segment_mapping_context *context,
 }
 
 /*
+ * check hdfs file length equals to pg_aoseg file logic length
+ */
+static void double_check_hdfs_metadata_logic_length(BlockLocation * locations,int block_num,int64
logic_len) {
+	//double check hdfs file length equals to pg_aoseg logic length
+	int64 hdfs_file_len = 0;
+	for(int i=0;i<block_num;i++) {
+		hdfs_file_len += locations[i].length;
+	}
+	if(logic_len != hdfs_file_len) {
+		elog(ERROR, "hdfs file length does not equal to metadata logic length!");
+	}
+}
+
+/*
  * AOGetSegFileDataLocation: fetch the data location of the
  * segment files of the AO relation.
  */
@@ -1254,6 +1272,7 @@ static void AOGetSegFileDataLocation(Relation relation,
 					NULL));
 			int64 logic_len = (int64) DatumGetFloat8(
 					fastgetattr(tuple, Anum_pg_aoseg_eof, pg_aoseg_dsc, NULL));
+			context->total_metadata_logic_len += logic_len;
 			bool isRelationHash = true;
 			if (targetPolicy->nattrs == 0) {
 				isRelationHash = false;
@@ -1269,6 +1288,7 @@ static void AOGetSegFileDataLocation(Relation relation,
 				if ((locations != NULL) && (block_num > 0)) {
 					Block_Host_Index * host_index = update_data_dist_stat(context,
 							locations, block_num);
+					double_check_hdfs_metadata_logic_length(locations, block_num, logic_len);
 
 					file = (Relation_File *) palloc(sizeof(Relation_File));
 					file->segno = segno;
@@ -1344,6 +1364,7 @@ static void AOGetSegFileDataLocation(Relation relation,
 				if ((locations != NULL) && (block_num > 0)) {
 					Block_Host_Index * host_index = update_data_dist_stat(context,
 							locations, block_num);
+					double_check_hdfs_metadata_logic_length(locations, block_num, logic_len);
 					// fake data locality
 					if (debug_fake_datalocality) {
 						for (int k = 0; k < block_num; k++) {
@@ -1426,7 +1447,7 @@ static void ParquetGetSegFileDataLocation(Relation relation,
 				fastgetattr(tuple, Anum_pg_parquetseg_segno, pg_parquetseg_dsc, NULL));
 		int64 logic_len = (int64) DatumGetFloat8(
 				fastgetattr(tuple, Anum_pg_parquetseg_eof, pg_parquetseg_dsc, NULL));
-
+		context->total_metadata_logic_len += logic_len;
 		bool isRelationHash = true;
 		if (targetPolicy->nattrs == 0) {
 			isRelationHash = false;
@@ -1456,6 +1477,8 @@ static void ParquetGetSegFileDataLocation(Relation relation,
 				Block_Host_Index * host_index = update_data_dist_stat(context,
 						locations, block_num);
 
+				double_check_hdfs_metadata_logic_length(locations, block_num, logic_len);
+
 				file = (Relation_File *) palloc(sizeof(Relation_File));
 				file->segno = segno;
 				file->block_num = block_num;
@@ -1509,6 +1532,8 @@ static void ParquetGetSegFileDataLocation(Relation relation,
 				Block_Host_Index * host_index = update_data_dist_stat(context,
 						locations, block_num);
 
+				double_check_hdfs_metadata_logic_length(locations, block_num, logic_len);
+
 				file->block_num = block_num;
 				file->locations = locations;
 				file->hostIDs = host_index;
@@ -3464,6 +3489,7 @@ static void combine_all_splits(Detailed_File_Split **splits,
 			sizeof(Detailed_File_Split) * assignment_context->total_split_num);
 	int total_split_index = 0;
 	bool nonLocalExist = false;
+	int64 splitTotalLength = 0;
 	/* go through all splits again. combine all splits to Detailed_File_Split structure*/
 	foreach(lc, context->chsl_context.relations)
 	{
@@ -3555,9 +3581,13 @@ static void combine_all_splits(Detailed_File_Split **splits,
 				(*splits)[total_split_index].length = rel_file->splits[i].length;
 				(*splits)[total_split_index].logiceof = rel_file->logic_len;
 				total_split_index++;
+				splitTotalLength += rel_file->splits[i].length;
 			}
 		}
 	}
+	if(context->total_metadata_logic_len != splitTotalLength){
+		elog(ERROR, "total split length does not equal to metadata total logic length!");
+	}
 }
 
 /*


Mime
View raw message