hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m..@apache.org
Subject incubator-hawq git commit: HAWQ-1345. Fixed relfile path bug: catalog and hdfs relfile have different rule
Date Tue, 21 Feb 2017 01:55:07 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master f5033eaa3 -> f77a332fb


HAWQ-1345. Fixed relfile path bug: catalog and hdfs relfile have different rule

Signed-off-by: Amy Bai <abai@pivotal.io>


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f77a332f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f77a332f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f77a332f

Branch: refs/heads/master
Commit: f77a332fb97346144f06f776ea0e59dbf7fe4ca7
Parents: f5033ea
Author: Ming LI <mli@apache.org>
Authored: Mon Feb 20 17:50:07 2017 +0800
Committer: Amy Bai <abai@pivotal.io>
Committed: Tue Feb 21 00:11:39 2017 +0800

----------------------------------------------------------------------
 src/backend/access/appendonly/aomd.c    |  7 +---
 src/backend/catalog/catalog.c           | 45 +++++++++++++++++++++++
 src/backend/cdb/cdbmirroredappendonly.c | 55 ++++++----------------------
 src/backend/cdb/cdbmirroredbufferpool.c | 35 ++++--------------
 src/include/catalog/catalog.h           |  1 +
 5 files changed, 67 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f77a332f/src/backend/access/appendonly/aomd.c
----------------------------------------------------------------------
diff --git a/src/backend/access/appendonly/aomd.c b/src/backend/access/appendonly/aomd.c
index 4e236ff..b640019 100644
--- a/src/backend/access/appendonly/aomd.c
+++ b/src/backend/access/appendonly/aomd.c
@@ -101,12 +101,7 @@ FormatAOSegmentFileName(
 	
 	*fileSegNo = pseudoSegNo;
 
-	if (pseudoSegNo > 0)
-	{
-		sprintf(filepathname, "%s/%u", basepath, pseudoSegNo);
-	}
-	else
-		sprintf(filepathname, "%s", basepath);
+	sprintf(filepathname, "%s/%u", basepath, pseudoSegNo);
 }
 
 /*

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f77a332f/src/backend/catalog/catalog.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/catalog.c b/src/backend/catalog/catalog.c
index 45aa9bb..d4b1531 100755
--- a/src/backend/catalog/catalog.c
+++ b/src/backend/catalog/catalog.c
@@ -528,6 +528,51 @@ FormRelationPath(char *relationPath, char *filespaceLocation, RelFileNode
rnode)
 					targetMaxLen);
 }
 
+/**
+ * Form path for relfile, it support not only catalog relfile but also relfile on hdfs
+ */
+void
+FormRelfilePath(char *relfilePath, char *filespaceLocation, RelFileNode *rnode, int32 segmentFileNum)
+{
+	char		dbPath[MAXPGPATH + 1];
+	int			targetMaxLen = MAXPGPATH + 1;
+	int 		snprintfResult;
+
+	FormDatabasePath(dbPath,
+					 filespaceLocation,
+					 rnode->spcNode,
+					 rnode->dbNode);
+
+	if (IsLocalPath(dbPath))
+	{
+		/* catalog relfile rule same as postgres: all relfile at same level */
+		if (segmentFileNum == 0)
+			snprintfResult =
+						snprintf(relfilePath, targetMaxLen, "%s/%u", dbPath, rnode->relNode);
+		else
+			snprintfResult =
+						snprintf(relfilePath, targetMaxLen, "%s/%u.%u", dbPath, rnode->relNode, segmentFileNum);
+
+	}else
+	{
+		/* hdfs relfile rule change from file to directory at relation level, and  segnum */
+		snprintfResult =
+					snprintf(relfilePath, targetMaxLen, "%s/%u/%u", dbPath, rnode->relNode, segmentFileNum);
+	}
+
+	if (snprintfResult < 0)
+		elog(ERROR, "FormRelfilePath formatting error");
+
+	/*
+	 * Magically truncating the result to fit in the target string is unacceptable here
+	 * because it can result in the wrong file-system object being referenced.
+	 */
+	if (snprintfResult >= targetMaxLen)
+		elog(ERROR, "FormRelfilePath formatting result length %d exceeded the maximum length %d",
+					snprintfResult,
+					targetMaxLen);
+}
+
 /*
  * IsSystemRelation
  *		True iff the relation is a system catalog relation.

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f77a332f/src/backend/cdb/cdbmirroredappendonly.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbmirroredappendonly.c b/src/backend/cdb/cdbmirroredappendonly.c
index b0c3c76..43fed80 100644
--- a/src/backend/cdb/cdbmirroredappendonly.c
+++ b/src/backend/cdb/cdbmirroredappendonly.c
@@ -95,26 +95,18 @@ static void MirroredAppendOnly_DoOpen(
 										TRUE,
 										&filespaceLocation);
 
-	char *dbPath;
 	char *path;
 
-	dbPath = (char*)palloc(MAXPGPATH + 1);
 	path = (char*)palloc(MAXPGPATH + 1);
 
 	/*
 	 * Do the primary work first so we don't leave files on the mirror or have an
 	 * open to clean up.
 	 */
-	FormDatabasePath(
-				dbPath,
-				filespaceLocation,
-				relFileNode->spcNode,
-				relFileNode->dbNode);
-	
-	if (segmentFileNum == 0)
-		sprintf(path, "%s/%u", dbPath, relFileNode->relNode);
-	else
-		sprintf(path, "%s/%u/%u", dbPath, relFileNode->relNode, segmentFileNum);
+	FormRelfilePath(path,
+					 filespaceLocation,
+					 relFileNode,
+					 segmentFileNum);
 
 	errno = 0;
 
@@ -133,7 +125,6 @@ static void MirroredAppendOnly_DoOpen(
 		*primaryError = errno;
 	}
 
-	pfree(dbPath);
 	pfree(path);
 
 	if (*primaryError != 0)
@@ -164,27 +155,14 @@ void AppendOnly_Overwrite(RelFileNode *relFileNode, int32 segmentFileNum,
int *p
 										TRUE,
 										&primaryFilespaceLocation);
 
-	char *dbPath;
 	char *path;
 
-	dbPath = (char*)palloc(MAXPGPATH + 1);
 	path = (char*)palloc(MAXPGPATH + 1);
 
-	/*
-	 * Do the primary work first so we don't leave files on the mirror or have an
-	 * open to clean up.
-	 */
-	FormDatabasePath(
-				dbPath,
-				primaryFilespaceLocation,
-				relFileNode->spcNode,
-				relFileNode->dbNode);
-
-	if (segmentFileNum == 0)
-		sprintf(path, "%s/%u", dbPath, relFileNode->relNode);
-	else
-		sprintf(path, "%s/%u/%u", dbPath, relFileNode->relNode, segmentFileNum);
-
+	FormRelfilePath(path,
+					 primaryFilespaceLocation,
+					 relFileNode,
+					 segmentFileNum);
 	errno = 0;
 
 	fileFlags = O_WRONLY | O_SYNC;
@@ -194,7 +172,6 @@ void AppendOnly_Overwrite(RelFileNode *relFileNode, int32 segmentFileNum,
int *p
 	if (file < 0)
 		*primaryError = errno;
 
-	pfree(dbPath);
 	pfree(path);
 
 	if (primaryFilespaceLocation != NULL)
@@ -389,22 +366,15 @@ static void MirroredAppendOnly_DoDrop(
 
     HdfsFileInfo *file_info;
 
-	char *dbPath;
+
 	char *path;
 
-	dbPath = (char*)palloc(MAXPGPATH + 1);
 	path = (char*)palloc(MAXPGPATH + 1);
 
-	FormDatabasePath(
-					 dbPath,
+	FormRelfilePath(path,
 					 filespaceLocation,
-					 relFileNode->spcNode,
-					 relFileNode->dbNode);
-
-	if (segmentFileNum == 0)
-		sprintf(path, "%s/%u", dbPath, relFileNode->relNode);
-	else
-		sprintf(path, "%s/%u/%u", dbPath, relFileNode->relNode, segmentFileNum);
+					 relFileNode,
+					 segmentFileNum);
 
 	errno = 0;
 	
@@ -423,7 +393,6 @@ static void MirroredAppendOnly_DoDrop(
         DestroyHdfsFileInfo(file_info);
     }
 
-	pfree(dbPath);
 	pfree(path);
 
 	if (filespaceLocation != NULL)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f77a332f/src/backend/cdb/cdbmirroredbufferpool.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbmirroredbufferpool.c b/src/backend/cdb/cdbmirroredbufferpool.c
index 8799912..0dea717 100644
--- a/src/backend/cdb/cdbmirroredbufferpool.c
+++ b/src/backend/cdb/cdbmirroredbufferpool.c
@@ -117,27 +117,18 @@ static void MirroredBufferPool_DoOpen(
 	open->create = create;
 	if (true)
 	{
-		char *dbPath;
 		char *path;
 
-		dbPath = (char*)palloc(MAXPGPATH + 1);
 		path = (char*)palloc(MAXPGPATH + 1);
 
 		/*
 		 * Do the primary work first so we don't leave files on the mirror or have an
 		 * open to clean up.
 		 */
-
-		FormDatabasePath(
-					dbPath,
-					filespaceLocation,
-					relFileNode->spcNode,
-					relFileNode->dbNode);
-		
-		if (segmentFileNum == 0)
-			sprintf(path, "%s/%u", dbPath, relFileNode->relNode);
-		else
-			sprintf(path, "%s/%u/%u", dbPath, relFileNode->relNode, segmentFileNum);
+		FormRelfilePath(path,
+						filespaceLocation,
+						relFileNode,
+						segmentFileNum);
 
 		errno = 0;
 		
@@ -148,7 +139,6 @@ static void MirroredBufferPool_DoOpen(
 			*primaryError = errno;
 		}
 
-		pfree(dbPath);
 		pfree(path);
 	}
 	
@@ -454,22 +444,14 @@ static void MirroredBufferPool_DoDrop(
 	
 	if (true)
 	{
-		char *dbPath; 
 		char *path;
 
-		dbPath = (char*)palloc(MAXPGPATH + 1);
 		path = (char*)palloc(MAXPGPATH + 1);
 
-		FormDatabasePath(
-						 dbPath,
-						 filespaceLocation,
-						 relFileNode->spcNode,
-						 relFileNode->dbNode);
-		
-		if (segmentFileNum == 0)
-			sprintf(path, "%s/%u", dbPath, relFileNode->relNode);
-		else
-			sprintf(path, "%s/%u/%u", dbPath, relFileNode->relNode, segmentFileNum);
+		FormRelfilePath(path,
+						filespaceLocation,
+						relFileNode,
+						segmentFileNum);
 
 		errno = 0;
 		
@@ -478,7 +460,6 @@ static void MirroredBufferPool_DoDrop(
 			*primaryError = errno;
 		}
 
-		pfree(dbPath);
 		pfree(path);
 	}
 	

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f77a332f/src/include/catalog/catalog.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h
index d8c7e1e..045ebd3 100755
--- a/src/include/catalog/catalog.h
+++ b/src/include/catalog/catalog.h
@@ -26,6 +26,7 @@ extern void CopyDatabasePath(char *target, int targetMaxLen, Oid dbNode,
Oid spc
 extern void FormDatabasePath(char *databasePath, char *filespaceLocation, Oid tablespaceOid,
Oid databaseOid);
 extern void FormTablespacePath(char *tablespacePath, char *filespaceLocation, Oid tablespaceOid);
 extern void FormRelationPath(char *relationPath, char *filespaceLocation, RelFileNode rnode);
+extern void FormRelfilePath(char *relfilePath, char *filespaceLocation, RelFileNode *rnode,
int32 segmentFileNum);
 
 extern bool IsSystemRelation(Relation relation);
 extern bool IsToastRelation(Relation relation);


Mime
View raw message