hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m..@apache.org
Subject incubator-hawq git commit: HAWQ-232: recovery Pass3 do not need to redo all relfile and relation dir create/drop for on flight transaction
Date Thu, 10 Dec 2015 06:14:31 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master e66f9ecc6 -> b845551ed


HAWQ-232: recovery Pass3 do not need to redo all relfile and relation dir create/drop for
on flight transaction


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b845551e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b845551e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b845551e

Branch: refs/heads/master
Commit: b845551ed1090b3f13bb4e3bfa5d44c337f6e27d
Parents: e66f9ec
Author: Ming LI <mli@pivotal.io>
Authored: Mon Dec 7 19:02:36 2015 +0800
Committer: Ming LI <mli@pivotal.io>
Committed: Thu Dec 10 14:09:14 2015 +0800

----------------------------------------------------------------------
 .../access/transam/persistentfilesysobjname.c   |  2 +-
 src/backend/access/transam/xlog_mm.c            | 48 ++++++++++++++++----
 src/backend/cdb/cdbpersistentdatabase.c         |  9 ++--
 src/backend/cdb/cdbpersistentfilespace.c        |  4 +-
 src/backend/cdb/cdbpersistentrelation.c         |  6 ++-
 src/backend/cdb/cdbpersistentrelfile.c          |  6 ++-
 src/backend/cdb/cdbpersistenttablespace.c       |  5 +-
 .../cdb/cdbresynchronizechangetracking.c        | 27 ++++++++++-
 src/include/access/xlogmm.h                     | 24 ++++++----
 9 files changed, 98 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b845551e/src/backend/access/transam/persistentfilesysobjname.c
----------------------------------------------------------------------
diff --git a/src/backend/access/transam/persistentfilesysobjname.c b/src/backend/access/transam/persistentfilesysobjname.c
index 60627f6..bc3dd8a 100755
--- a/src/backend/access/transam/persistentfilesysobjname.c
+++ b/src/backend/access/transam/persistentfilesysobjname.c
@@ -67,7 +67,7 @@ char *PersistentFileSysObjName_ObjectName(
 		else
 		{
 			/* All other tablespaces are accessed via filespace locations */
-			sprintf(path, "%u/%u/%u%u",
+			sprintf(path, "%u/%u/%u/%u",
 					 name->variant.rel.relFileNode.spcNode,
 					 name->variant.rel.relFileNode.dbNode,
 					 name->variant.rel.relFileNode.relNode,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b845551e/src/backend/access/transam/xlog_mm.c
----------------------------------------------------------------------
diff --git a/src/backend/access/transam/xlog_mm.c b/src/backend/access/transam/xlog_mm.c
index 938eb2d..bface0c 100644
--- a/src/backend/access/transam/xlog_mm.c
+++ b/src/backend/access/transam/xlog_mm.c
@@ -437,6 +437,7 @@ append_file_parts(mm_fs_obj_type type,
 static bool
 emit_mmxlog_fs_record(mm_fs_obj_type type, Oid filespace,
 					  Oid tablespace, Oid database, Oid relfilenode,
+					  ItemPointer persistentTid, int64 persistentSerialNum,
 					  int32 segnum, uint8 flags, XLogRecPtr *beginLoc)
 {
 	XLogRecData		rdata;
@@ -480,6 +481,8 @@ emit_mmxlog_fs_record(mm_fs_obj_type type, Oid filespace,
 	xlrec.relfilenode = relfilenode;
 	xlrec.segnum = segnum;
 	xlrec.shared = is_filespace_shared(filespace);
+    xlrec.persistentTid = *persistentTid;
+    xlrec.persistentSerialNum = persistentSerialNum;
 
 	Insist(!path || strlen(path) <= MAXPGPATH);
 
@@ -516,7 +519,7 @@ emit_mmxlog_fs_record(mm_fs_obj_type type, Oid filespace,
 
 /* External interface to filespace removal logging */
 void
-mmxlog_log_remove_filespace(Oid filespace)
+mmxlog_log_remove_filespace(Oid filespace,ItemPointer persistentTid, int64 persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -531,6 +534,8 @@ mmxlog_log_remove_filespace(Oid filespace)
 						  InvalidOid /* tablespace */,
 						  InvalidOid /* database */,
 						  InvalidOid /* relfilenode */,
+						  persistentTid,
+						  persistentSerialNum,
 						  0 /* segnum */,
 						  MMXLOG_REMOVE_DIR,
 						  &beginLoc);
@@ -552,7 +557,7 @@ mmxlog_log_remove_filespace(Oid filespace)
 
 /* External interface to tablespace removal logging */
 void
-mmxlog_log_remove_tablespace(Oid tablespace)
+mmxlog_log_remove_tablespace(Oid tablespace,ItemPointer persistentTid, int64 persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -567,6 +572,8 @@ mmxlog_log_remove_tablespace(Oid tablespace)
 						  tablespace,
 						  InvalidOid /* database */,
 						  InvalidOid /* relfilenode */,
+						  persistentTid,
+						  persistentSerialNum,
 						  0 /* segnum */,
 						  MMXLOG_REMOVE_DIR,
 						  &beginLoc);
@@ -588,7 +595,8 @@ mmxlog_log_remove_tablespace(Oid tablespace)
 
 /* External interface to database removal logging */
 void
-mmxlog_log_remove_database(Oid tablespace, Oid database)
+mmxlog_log_remove_database(Oid tablespace, Oid database,
+		ItemPointer persistentTid, int64 persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -603,6 +611,8 @@ mmxlog_log_remove_database(Oid tablespace, Oid database)
 						  tablespace,
 						  database,
 						  InvalidOid /* relfilenode */,
+						  persistentTid,
+						  persistentSerialNum,
 						  0 /* segnum */,
 						  MMXLOG_REMOVE_DIR,
 						  &beginLoc);
@@ -625,7 +635,8 @@ mmxlog_log_remove_database(Oid tablespace, Oid database)
 
 /* External interface to relation removal logging */
 void
-mmxlog_log_remove_relation(Oid tablespace, Oid database, Oid relfilenode)
+mmxlog_log_remove_relation(Oid tablespace, Oid database, Oid relfilenode,
+		ItemPointer persistentTid, int64 persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -642,6 +653,8 @@ mmxlog_log_remove_relation(Oid tablespace, Oid database, Oid relfilenode)
 								tablespace,
 								database,
 								relfilenode,
+								persistentTid,
+								persistentSerialNum,
 								0 /* segnum */,
 								MMXLOG_REMOVE_DIR,
 								&beginLoc);
@@ -666,7 +679,7 @@ mmxlog_log_remove_relation(Oid tablespace, Oid database, Oid relfilenode)
 /* External interface to relfilenode removal logging */
 void
 mmxlog_log_remove_relfilenode(Oid tablespace, Oid database, Oid relfilenode,
-							  int32 segnum)
+							  int32 segnum,ItemPointer persistentTid, int64 persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -681,6 +694,8 @@ mmxlog_log_remove_relfilenode(Oid tablespace, Oid database, Oid relfilenode,
 						  tablespace,
 						  database,
 						  relfilenode,
+						  persistentTid,
+						  persistentSerialNum,
 						  segnum,
 						  MMXLOG_REMOVE_FILE,
 						  &beginLoc);
@@ -705,7 +720,7 @@ mmxlog_log_remove_relfilenode(Oid tablespace, Oid database, Oid relfilenode,
 
 /* External interface to filespace creation logging */
 void
-mmxlog_log_create_filespace(Oid filespace)
+mmxlog_log_create_filespace(Oid filespace,ItemPointer persistentTid, int64 persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -717,6 +732,8 @@ mmxlog_log_create_filespace(Oid filespace)
 						  InvalidOid /* tablespace */,
 						  InvalidOid /* database */,
 						  InvalidOid /* relfilenode */,
+						  persistentTid,
+						  persistentSerialNum,
 						  0 /* segnum */,
 						  MMXLOG_CREATE_DIR,
 						  &beginLoc);
@@ -738,7 +755,7 @@ mmxlog_log_create_filespace(Oid filespace)
 
 /* External interface to tablespace creation logging */
 void
-mmxlog_log_create_tablespace(Oid filespace, Oid tablespace)
+mmxlog_log_create_tablespace(Oid filespace, Oid tablespace, ItemPointer persistentTid, int64
persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -750,6 +767,8 @@ mmxlog_log_create_tablespace(Oid filespace, Oid tablespace)
 						  tablespace,
 						  InvalidOid /* database */,
 						  InvalidOid /* relfilenode */,
+						  persistentTid,
+						  persistentSerialNum,
 						  0 /* segnum */,
 						  MMXLOG_CREATE_DIR,
 						  &beginLoc);
@@ -772,7 +791,8 @@ mmxlog_log_create_tablespace(Oid filespace, Oid tablespace)
 
 /* External interface to database creation logging */
 void
-mmxlog_log_create_database(Oid tablespace, Oid database)
+mmxlog_log_create_database(Oid tablespace, Oid database,
+		ItemPointer persistentTid, int64 persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -784,6 +804,8 @@ mmxlog_log_create_database(Oid tablespace, Oid database)
 						  tablespace,
 						  database,
 						  InvalidOid /* relfilenode */,
+						  persistentTid,
+						  persistentSerialNum,
 						  0 /* segnum */,
 						  MMXLOG_CREATE_DIR,
 						  &beginLoc);
@@ -808,7 +830,8 @@ mmxlog_log_create_database(Oid tablespace, Oid database)
  * External interface to relation create logging
  */
 void
-mmxlog_log_create_relation(Oid tablespace, Oid database, Oid relfilenode)
+mmxlog_log_create_relation(Oid tablespace, Oid database, Oid relfilenode,
+		ItemPointer persistentTid, int64 persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -819,6 +842,8 @@ mmxlog_log_create_relation(Oid tablespace, Oid database, Oid relfilenode)
 								tablespace,
 								database,
 								relfilenode,
+								persistentTid,
+								persistentSerialNum,
 								0 /*segnum */,
 								MMXLOG_CREATE_DIR,
 								&beginLoc);
@@ -843,7 +868,8 @@ mmxlog_log_create_relation(Oid tablespace, Oid database, Oid relfilenode)
 /* External interface to relfilenode creation logging */
 void
 mmxlog_log_create_relfilenode(Oid tablespace, Oid database,
-							  Oid relfilenode, int32 segnum)
+							  Oid relfilenode, int32 segnum,
+							  ItemPointer persistentTid, int64 persistentSerialNum)
 {
 	bool emitted;
 	XLogRecPtr beginLoc;
@@ -855,6 +881,8 @@ mmxlog_log_create_relfilenode(Oid tablespace, Oid database,
 						  tablespace,
 						  database,
 						  relfilenode,
+						  persistentTid,
+						  persistentSerialNum,
 						  segnum,
 						  MMXLOG_CREATE_FILE,
 						  &beginLoc);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b845551e/src/backend/cdb/cdbpersistentdatabase.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbpersistentdatabase.c b/src/backend/cdb/cdbpersistentdatabase.c
index b52dafc..a78ca21 100644
--- a/src/backend/cdb/cdbpersistentdatabase.c
+++ b/src/backend/cdb/cdbpersistentdatabase.c
@@ -611,7 +611,8 @@ void PersistentDatabase_MarkCreatePending(
 	 * This XLOG must be generated under the persistent write-lock.
 	 */
 #ifdef MASTER_MIRROR_SYNC
-	mmxlog_log_create_database(dbDirNode->tablespace, dbDirNode->database); 
+	mmxlog_log_create_database(dbDirNode->tablespace, dbDirNode->database,
+			persistentTid, persistentSerialNum);
 #endif
 
 
@@ -1053,7 +1054,8 @@ PersistentDatabase_DroppedVerifiedActionCallback(
 		 * This XLOG must be generated under the persistent write-lock.
 		 */
 #ifdef MASTER_MIRROR_SYNC
-		mmxlog_log_remove_database(dbDirNode->tablespace, dbDirNode->database);
+		mmxlog_log_remove_database(dbDirNode->tablespace, dbDirNode->database,
+				persistentTid, persistentSerialNum);
 #endif
 				
 		break;
@@ -1288,7 +1290,8 @@ void PersistentDatabase_MarkJustInTimeCreatePending(
 	 */
 #ifdef MASTER_MIRROR_SYNC
 	mmxlog_log_create_database(dbDirNode->tablespace,
-							   dbDirNode->database);	
+							   dbDirNode->database,
+							   persistentTid, persistentSerialNum);
 #endif
 	
 	WRITE_PERSISTENT_STATE_ORDERED_UNLOCK;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b845551e/src/backend/cdb/cdbpersistentfilespace.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbpersistentfilespace.c b/src/backend/cdb/cdbpersistentfilespace.c
index b9a3a07..58418ef 100644
--- a/src/backend/cdb/cdbpersistentfilespace.c
+++ b/src/backend/cdb/cdbpersistentfilespace.c
@@ -612,7 +612,7 @@ void PersistentFilespace_MarkCreatePending(
 	 * This XLOG must be generated under the persistent write-lock.
 	 */
 #ifdef MASTER_MIRROR_SYNC
-	mmxlog_log_create_filespace(filespaceOid);
+	mmxlog_log_create_filespace(filespaceOid,persistentTid, persistentSerialNum);
 #endif
 
 
@@ -1029,7 +1029,7 @@ PersistentFilespace_DroppedVerifiedActionCallback(
 		 * This XLOG must be generated under the persistent write-lock.
 		 */
 #ifdef MASTER_MIRROR_SYNC
-		mmxlog_log_remove_filespace(filespaceOid);
+		mmxlog_log_remove_filespace(filespaceOid,persistentTid, persistentSerialNum);
 #endif
 
 		break;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b845551e/src/backend/cdb/cdbpersistentrelation.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbpersistentrelation.c b/src/backend/cdb/cdbpersistentrelation.c
index 7ec8958..e7fd7ed 100644
--- a/src/backend/cdb/cdbpersistentrelation.c
+++ b/src/backend/cdb/cdbpersistentrelation.c
@@ -628,7 +628,8 @@ void PersistentRelation_MarkCreatePending(
 	mmxlog_log_create_relation(
 						relFileNode->spcNode,
 						relFileNode->dbNode,
-						relFileNode->relNode);
+						relFileNode->relNode,
+						persistentTid, persistentSerialNum);
 #endif
 
 	/*
@@ -1046,7 +1047,8 @@ PersistentRelation_DroppedVerifiedActionCallback(
 		 * This XLOG must be generated under the persistent write-lock.
 		 */
 #ifdef MASTER_MIRROR_SYNC
-		mmxlog_log_remove_relation(relFileNode->spcNode, relFileNode->dbNode, relFileNode->relNode);
+		mmxlog_log_remove_relation(relFileNode->spcNode, relFileNode->dbNode, relFileNode->relNode,
+				persistentTid, persistentSerialNum);
 #endif
 		break;
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b845551e/src/backend/cdb/cdbpersistentrelfile.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbpersistentrelfile.c b/src/backend/cdb/cdbpersistentrelfile.c
index 8c5466e..e7c2d63 100644
--- a/src/backend/cdb/cdbpersistentrelfile.c
+++ b/src/backend/cdb/cdbpersistentrelfile.c
@@ -324,7 +324,8 @@ void PersistentRelfile_AddCreatePending(
 						relFileNode->spcNode,
 						relFileNode->dbNode,
 						relFileNode->relNode,
-						segmentFileNum);	
+						segmentFileNum,
+						persistentTid, serialNum);
 #endif
 
 	#ifdef FAULT_INJECTOR
@@ -826,7 +827,8 @@ PersistentRelfile_DroppedVerifiedActionCallback(
 							relFileNode->spcNode, 
 							relFileNode->dbNode, 
 							relFileNode->relNode,
-							segmentFileNum);
+							segmentFileNum,
+							persistentTid, persistentSerialNum);
 #endif
 				
 		break;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b845551e/src/backend/cdb/cdbpersistenttablespace.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbpersistenttablespace.c b/src/backend/cdb/cdbpersistenttablespace.c
index 5a008e9..e7ddc33 100644
--- a/src/backend/cdb/cdbpersistenttablespace.c
+++ b/src/backend/cdb/cdbpersistenttablespace.c
@@ -711,7 +711,8 @@ void PersistentTablespace_MarkCreatePending(
 #ifdef MASTER_MIRROR_SYNC
 	mmxlog_log_create_tablespace(
 						filespaceOid,
-						tablespaceOid);
+						tablespaceOid,
+						persistentTid, persistentSerialNum);
 #endif
 
 	#ifdef FAULT_INJECTOR
@@ -1075,7 +1076,7 @@ PersistentTablespace_DroppedVerifiedActionCallback(
 		 * This XLOG must be generated under the persistent write-lock.
 		 */
 #ifdef MASTER_MIRROR_SYNC
-		mmxlog_log_remove_tablespace(tablespaceOid);
+		mmxlog_log_remove_tablespace(tablespaceOid,persistentTid, persistentSerialNum);
 #endif
 				
 		break;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b845551e/src/backend/cdb/cdbresynchronizechangetracking.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbresynchronizechangetracking.c b/src/backend/cdb/cdbresynchronizechangetracking.c
index f8361b6..e25ad0a 100644
--- a/src/backend/cdb/cdbresynchronizechangetracking.c
+++ b/src/backend/cdb/cdbresynchronizechangetracking.c
@@ -197,9 +197,34 @@ void ChangeTracking_GetRelationChangeInfoFromXlog(
 		case RM_SMGR_ID:
 		case RM_DBASE_ID:
 		case RM_TBLSPC_ID:
+			break;
 		case RM_MMXLOG_ID:
+			switch (info)
+			{
+				// relation dirs and node files create/drop need to be tracked down
+                // so that we can skip at recovery pass3
+                case MMXLOG_CREATE_DIR:
+				case MMXLOG_CREATE_FILE:
+                case MMXLOG_REMOVE_DIR:
+				case MMXLOG_REMOVE_FILE:
+				{
+					xl_heap_freeze *xlrec = (xl_heap_freeze *) data;
+
+					ChangeTracking_AddRelationChangeInfo(
+													   relationChangeInfoArray,
+													   relationChangeInfoArrayCount,
+													   relationChangeInfoMaxSize,
+													   &(xlrec->heapnode.node),
+													   InvalidBlockNumber,
+													   &xlrec->heapnode.persistentTid,
+													   xlrec->heapnode.persistentSerialNum);
+					break;
+				}
+				default:
+					break;
+			}
 			break;
-			
+
 		/* 
 		 * These aren't supported in GPDB
 		 */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b845551e/src/include/access/xlogmm.h
----------------------------------------------------------------------
diff --git a/src/include/access/xlogmm.h b/src/include/access/xlogmm.h
index 4930d12..1d4b9c5 100644
--- a/src/include/access/xlogmm.h
+++ b/src/include/access/xlogmm.h
@@ -12,6 +12,7 @@
  */
 #define MASTER_MIRROR_SYNC
 
+#include "c.h"
 #include "access/xlog.h"
 #include "access/xlogdefs.h"
 #include "lib/stringinfo.h"
@@ -42,6 +43,9 @@ typedef struct xl_mm_fs_obj
 
 	bool shared;
 	char path[MAXPGPATH];
+    
+	ItemPointerData persistentTid;
+	int64 persistentSerialNum;
 } xl_mm_fs_obj;
 
 /*
@@ -120,23 +124,23 @@ extern void mmxlog_desc(StringInfo buf, XLogRecPtr beginLoc, XLogRecord
*record)
  * Functions to generate WAL records to remove file system objects on the
  * master / standby master.
  */
-extern void mmxlog_log_remove_filespace(Oid filespace);
-extern void mmxlog_log_remove_tablespace(Oid tablespace);
-extern void mmxlog_log_remove_database(Oid tablespace, Oid database);
-extern void mmxlog_log_remove_relation(Oid tablespace, Oid database, Oid relfilenode);
+extern void mmxlog_log_remove_filespace(Oid filespace,ItemPointer persistentTid, int64 persistentSerialNum);
+extern void mmxlog_log_remove_tablespace(Oid tablespace,ItemPointer persistentTid, int64
persistentSerialNum);
+extern void mmxlog_log_remove_database(Oid tablespace, Oid database,ItemPointer persistentTid,
int64 persistentSerialNum);
+extern void mmxlog_log_remove_relation(Oid tablespace, Oid database, Oid relfilenode,ItemPointer
persistentTid, int64 persistentSerialNum);
 extern void mmxlog_log_remove_relfilenode(Oid tablespace, Oid database,
-										  Oid relfilenode, int32 segnum);
+										  Oid relfilenode, int32 segnum,ItemPointer persistentTid, int64 persistentSerialNum);
 
 /*
  * Functions to generate WAL records to add file system objects on the
  * master / standby master.
  */
-extern void mmxlog_log_create_filespace(Oid filespace);
-extern void mmxlog_log_create_tablespace(Oid filespace, Oid tablespace);
-extern void mmxlog_log_create_database(Oid tablespace, Oid database);
-extern void mmxlog_log_create_relation(Oid tablespace, Oid database, Oid relfilenode);
+extern void mmxlog_log_create_filespace(Oid filespace,ItemPointer persistentTid, int64 persistentSerialNum);
+extern void mmxlog_log_create_tablespace(Oid filespace, Oid tablespace,ItemPointer persistentTid,
int64 persistentSerialNum);
+extern void mmxlog_log_create_database(Oid tablespace, Oid database,ItemPointer persistentTid,
int64 persistentSerialNum);
+extern void mmxlog_log_create_relation(Oid tablespace, Oid database, Oid relfilenode,ItemPointer
persistentTid, int64 persistentSerialNum);
 extern void mmxlog_log_create_relfilenode(Oid tablespace, Oid database,
-										  Oid relfilenode, int32 segnum);
+										  Oid relfilenode, int32 segnum,ItemPointer persistentTid, int64 persistentSerialNum);
 extern void mmxlog_append_checkpoint_data(XLogRecData rdata[5]);
 extern void mmxlog_read_checkpoint_data(char *cpdata, int masterMirroringLen, int checkpointLen,
XLogRecPtr *beginLoc);
 extern bool mmxlog_filespace_get_path(


Mime
View raw message