hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m..@apache.org
Subject incubator-hawq git commit: HAWQ-1149. Fixed bug gp_persistent_build_all loses data in gp_relfile_node&gp_persistent_relfile_node
Date Mon, 21 Nov 2016 10:03:05 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 833c2f97c -> 38b447d7e


HAWQ-1149. Fixed bug gp_persistent_build_all loses data in gp_relfile_node&gp_persistent_relfile_node

Internally there are 3 bugs:
1) hawq2.0 changed the file path of relation file on hdfs from filespace/db/relfile.filenum
to filespace/db/table/relfile/filenum. When we scan relfile on hdfs, we should change scan
logic correspondingly.
2) Fetched dummy persistentTid & persistentSerialNum to PersistentRelation_MarkCreatePending()
.
3) Need to reset Relation->rd_relationnodeinfo.isPresent, so that next time persistentid
and serial# can be refetched during PersistentBuild_BuildDb().


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/38b447d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/38b447d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/38b447d7

Branch: refs/heads/master
Commit: 38b447d7e2a6a176bb41eefa7643ce14605a3f0b
Parents: 833c2f9
Author: Ming LI <mli@apache.org>
Authored: Mon Nov 14 16:21:46 2016 +0800
Committer: Ming LI <mli@apache.org>
Committed: Mon Nov 21 16:30:58 2016 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbdatabaseinfo.c       | 73 ++++++++++++++++++++++------
 src/backend/cdb/cdbpersistentbuild.c    |  6 +++
 src/backend/cdb/cdbpersistentrelation.c | 17 ++++---
 src/include/cdb/cdbpersistentrelation.h |  1 +
 4 files changed, 75 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/backend/cdb/cdbdatabaseinfo.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbdatabaseinfo.c b/src/backend/cdb/cdbdatabaseinfo.c
index 062c4d3..4e0be41 100644
--- a/src/backend/cdb/cdbdatabaseinfo.c
+++ b/src/backend/cdb/cdbdatabaseinfo.c
@@ -49,7 +49,6 @@
 #include "cdb/cdbpersistentfilesysobj.h"
 #include "commands/dbcommands.h"
 
-
 /*-------------------------------------------------------------------------
  * Local static type declarations
  *------------------------------------------------------------------------- */
@@ -753,7 +752,9 @@ static void DatabaseInfo_AddFile(
 	HTAB 					*dbInfoRelHashTable,
 	Oid 					tablespace,
 	char					*dbDirPath,
-	char					*name)
+	char					*name,
+	bool                    isHdfs,
+	Oid                     table_oid)
 {
 	int64 eof;
 	int itemCount;
@@ -787,7 +788,7 @@ static void DatabaseInfo_AddFile(
 	}
 	FileClose(file);
 
-	itemCount = sscanf(name, "%u.%u", &relfilenode, &segmentFileNum);
+	itemCount = sscanf(name, isHdfs?"%u/%u":"%u.%u", &relfilenode, &segmentFileNum);
 
 	// UNDONE: sscanf is a rather poor scanner.
 	// UNDONE: For right now, just assume properly named files....
@@ -796,12 +797,16 @@ static void DatabaseInfo_AddFile(
 		DatabaseInfo_AddMiscEntry(info, tablespace, false, name);
 		return;
 	}
-	else if (itemCount == 1)
+	else if (itemCount == 1 && !isHdfs)
 		segmentFileNum = 0;
 	else
 		Assert(itemCount == 2);
 
-	DatabaseInfo_AddRelSegFile(info, dbInfoRelHashTable, tablespace, relfilenode, segmentFileNum,
eof);
+	if(isHdfs){
+		DatabaseInfo_AddRelSegFile(info, dbInfoRelHashTable, tablespace, table_oid, relfilenode,
eof);
+	}else{
+		DatabaseInfo_AddRelSegFile(info, dbInfoRelHashTable, tablespace, relfilenode, segmentFileNum,
eof);
+	}
 }
 
 /*
@@ -818,9 +823,9 @@ DatabaseInfo_Scan(
 	Oid 				 database)
 {
 	char				*dbDirPath;
-	DIR					*xldir;
-	struct dirent		*xlde;
-	char				 fromfile[MAXPGPATH];
+	DIR					*xldir, *xldir1;
+	struct dirent		*xlde,*xlde1;
+	char				 fromfile[MAXPGPATH],fromfile1[MAXPGPATH];
 
 	/* Lookup the database path and allocate a directory scan structure */
 	dbDirPath = GetDatabasePath(
@@ -869,7 +874,9 @@ DatabaseInfo_Scan(
 									dbInfoRelHashTable,
 									tablespace,
 									dbDirPath,
-									xlde->d_name);
+									xlde->d_name,
+									false,
+									InvalidOid);
 			}
 		}else{//on hdfs
 			int ret = HdfsIsDirOrFile(fromfile);
@@ -878,18 +885,52 @@ DatabaseInfo_Scan(
 							(errcode_for_file_access(),
 							 errmsg("error to fetch info for path \"%s\": %m", fromfile)));
 			}else if(ret == 0){//direcotry
-				DatabaseInfo_AddMiscEntry(
+				if(strcmp(xlde->d_name, dbDirPath) == 0){ //database level
+					DatabaseInfo_AddMiscEntry(
 										info,
 										tablespace,
 										/* isDir */ true,
 										xlde->d_name);
+				}else{ //table level, need to recursive list file in this directory
+					xldir1 = AllocateDir(fromfile);
+					if (xldir1 == NULL)
+							ereport(ERROR,
+									(errcode_for_file_access(),
+									 errmsg("Could not open database directory \"%s\": %m", fromfile)));
+					while ((xlde1 = ReadDir(xldir1, fromfile)) != NULL)
+						{
+							if (strcmp(xlde1->d_name, ".") == 0 ||
+								strcmp(xlde1->d_name, "..") == 0)
+								continue;
+
+							/* Odd... On snow leopard, we get back "/" as a subdir, which is wrong. Ingore it
*/
+							if (xlde1->d_name[0] == '/' && xlde1->d_name[1] == '\0')
+								continue;
+
+							snprintf(fromfile1, MAXPGPATH, "%s/%s", fromfile, xlde1->d_name);
+
+							if(!IsLocalPath(fromfile1)){//on hdfs
+								int ret = HdfsIsDirOrFile(fromfile1);
+								if(ret != 1){//fetch info error
+										ereport(ERROR,
+													(errcode_for_file_access(),
+													 errmsg("error to fetch file path \"%s\": %m", fromfile1)));
+								}else{
+									DatabaseInfo_AddFile(
+														info,
+														dbInfoRelHashTable,
+														tablespace,
+														fromfile,
+														xlde1->d_name,
+														true,
+														pg_atoi(xlde->d_name, 4, 0));
+								}
+							}
+					}
+					FreeDir(xldir1);
+				}
 			}else if(ret == 1){//file
-				DatabaseInfo_AddFile(
-									info,
-									dbInfoRelHashTable,
-									tablespace,
-									dbDirPath,
-									xlde->d_name);
+				//just skip for PG_VERSION file
 			}
 		}
 	}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/backend/cdb/cdbpersistentbuild.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbpersistentbuild.c b/src/backend/cdb/cdbpersistentbuild.c
index 0033769..a12bc3b 100644
--- a/src/backend/cdb/cdbpersistentbuild.c
+++ b/src/backend/cdb/cdbpersistentbuild.c
@@ -222,6 +222,7 @@ static void PersistentBuild_PopulateGpRelationNode(
 
 		ItemPointerData persistentTid;
 		int64 persistentSerialNum;
+		Relation	rd;
 
 		if (dbInfoRel->reltablespace == GLOBALTABLESPACE_OID &&
 			info->database != TemplateDbOid)
@@ -417,6 +418,11 @@ static void PersistentBuild_PopulateGpRelationNode(
 								persistentSerialNum);
 			}
 		}
+		// reset Relation->rd_relationnodeinfo.isPresent, so that next time persistentid and
serial# can be refetched
+		rd = RelationIdGetRelation(relFileNode.relNode);
+		rd->rd_relationnodeinfo.isPresent = false;
+		RelationClose(rd);
+
 		(*count)++;
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/backend/cdb/cdbpersistentrelation.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbpersistentrelation.c b/src/backend/cdb/cdbpersistentrelation.c
index e7fd7ed..14a2cbc 100644
--- a/src/backend/cdb/cdbpersistentrelation.c
+++ b/src/backend/cdb/cdbpersistentrelation.c
@@ -629,7 +629,7 @@ void PersistentRelation_MarkCreatePending(
 						relFileNode->spcNode,
 						relFileNode->dbNode,
 						relFileNode->relNode,
-						persistentTid, persistentSerialNum);
+						persistentTid, *persistentSerialNum);
 #endif
 
 	/*
@@ -648,14 +648,16 @@ void PersistentRelation_MarkCreatePending(
 void PersistentRelation_AddCreated(
 		RelFileNode *relFileNode,
 		ItemPointer persistentTid,
+		int64 *persistentSerialNum,
 		bool flushToXLog)
 {
 	WRITE_PERSISTENT_STATE_ORDERED_LOCK_DECLARE;
 
 	PersistentFileSysObjName fsObjName;
-	ItemPointerData previousFreeTid;
-	int64 persistentSerialNum;
+
 	RelationDirEntry relationDirEntry;
+
+	ItemPointerData previousFreeTid;
 	Datum values[Natts_gp_persistent_relation_node];
 
 	if (RelFileNode_IsEmpty(relFileNode))
@@ -711,8 +713,11 @@ void PersistentRelation_AddCreated(
 								PersistentFsObjType_RelationDir,
 								values,
 								flushToXLog,
-								persistentTid,
-								&persistentSerialNum);
+								&relationDirEntry->persistentTid,
+								&relationDirEntry->persistentSerialNum);
+
+	*persistentTid = relationDirEntry->persistentTid;
+	*persistentSerialNum = relationDirEntry->persistentSerialNum;
 
 	WRITE_PERSISTENT_STATE_ORDERED_UNLOCK;
 
@@ -721,7 +726,7 @@ void PersistentRelation_AddCreated(
 		elog(Persistent_DebugPrintLevel(),
 			"Persistent relation: Add '%s', in state 'Created', serial number " INT64_FORMAT " at
TID %s",
 			PersistentFileSysObjName_ObjectName(&fsObjName),
-			persistentSerialNum,
+			*persistentSerialNum,
 			ItemPointerToString(persistentTid));
 	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/38b447d7/src/include/cdb/cdbpersistentrelation.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbpersistentrelation.h b/src/include/cdb/cdbpersistentrelation.h
index 292aaba..9f4e184 100644
--- a/src/include/cdb/cdbpersistentrelation.h
+++ b/src/include/cdb/cdbpersistentrelation.h
@@ -68,6 +68,7 @@ extern void PersistentRelation_AddCreated(
 		RelFileNode *relFileNode,
 					/* The tablespace, database, and relation OIDs for the create. */
 		ItemPointer persistentTid,
+		int64 *persistentSerialNum,
 		bool flushToXLog);
 
 // -----------------------------------------------------------------------------


Mime
View raw message