hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject incubator-hawq git commit: HAWQ-41. Add a GUC to set max open AO segemnt files
Date Tue, 01 Dec 2015 02:55:13 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 8268e669c -> be4aabb7c


HAWQ-41. Add a GUC to set max open AO segemnt files

The newly added GUC is max_appendonly_segfiles, which determines
the max number of appendonly relation segment files open
concurrently.

In a large cluster, we usually need to increase the value of this
GUC in order to support high concurrencies.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/be4aabb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/be4aabb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/be4aabb7

Branch: refs/heads/master
Commit: be4aabb7ce55527b5e64760d1f5531db170092e6
Parents: 8268e66
Author: Lirong Jian <jianlirong@gmail.com>
Authored: Mon Nov 30 15:50:00 2015 +0800
Committer: Lirong Jian <jianlirong@gmail.com>
Committed: Tue Dec 1 10:54:48 2015 +0800

----------------------------------------------------------------------
 .../access/appendonly/appendonlywriter.c        | 21 ++++++++++----------
 src/backend/utils/misc/guc.c                    | 10 ++++++++++
 src/include/access/appendonlywriter.h           |  8 +-------
 3 files changed, 22 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/be4aabb7/src/backend/access/appendonly/appendonlywriter.c
----------------------------------------------------------------------
diff --git a/src/backend/access/appendonly/appendonlywriter.c b/src/backend/access/appendonly/appendonlywriter.c
index 73d84ac..b53bf83 100644
--- a/src/backend/access/appendonly/appendonlywriter.c
+++ b/src/backend/access/appendonly/appendonlywriter.c
@@ -41,6 +41,7 @@
  * GUC variables
  */
 int		MaxAppendOnlyTables;		/* Max # of tables */
+int MaxAORelSegFileStatus;  /* Max # of AO relation segfiles */
 extern bool		filesystem_support_truncate;
 
 /*
@@ -180,7 +181,7 @@ AppendOnlyWriterShmemSize(void)
 	size = add_size(size, sizeof(AppendOnlyWriterData));
 
 	/* The size of the AOSegfileStatus pool */
-	size = add_size(size, mul_size((Size)MAX_AOREL_SEGFILE_STATUS, sizeof(AOSegfileStatus)));
+	size = add_size(size, mul_size((Size)MaxAORelSegFileStatus, sizeof(AOSegfileStatus)));
 
 	/* Add a safety margin */
 	size = add_size(size, size / 10);
@@ -224,7 +225,7 @@ InitAppendOnlyWriter(void)
 				 errmsg("not enough shared memory for append only writer")));
 
 	/* Create the pool of AOSegfileStatus. */
-	poolAOSegfileStatusTotalSize = mul_size((Size)MAX_AOREL_SEGFILE_STATUS, sizeof(AOSegfileStatus));
+	poolAOSegfileStatusTotalSize = mul_size((Size)MaxAORelSegFileStatus, sizeof(AOSegfileStatus));
 	AOSegfileStatusPool = (AOSegfileStatus *)
 			ShmemInitStruct("Append Only Segment Status Pool",
 					poolAOSegfileStatusTotalSize,
@@ -232,12 +233,12 @@ InitAppendOnlyWriter(void)
 
 	/* Initialize all segfile status */
 	status = AOSegfileStatusPool;
-	for(i = 0; i < MAX_AOREL_SEGFILE_STATUS; status++, i++)
+	for(i = 0; i < MaxAORelSegFileStatus; status++, i++)
 	{
 		status->id = i;
 		status->next = i + 1;
 	}
-	AOSegfileStatusPool[MAX_AOREL_SEGFILE_STATUS-1].next = NEXT_END_OF_LIST;
+	AOSegfileStatusPool[MaxAORelSegFileStatus-1].next = NEXT_END_OF_LIST;
 
 	/* Specify we have no open segment files now */
 	AppendOnlyWriter->head_free_segfilestatus = 0;
@@ -373,7 +374,7 @@ AORelCreateHashEntry(Oid relid)
 
 	Insist(aoHashEntry->relid == relid);
 	aoHashEntry->txns_using_rel = 0;
-    aoHashEntry->staleTid = InvalidTransactionId;
+  aoHashEntry->staleTid = InvalidTransactionId;
 
 	/*
 	 * Segment file number 0 is reserved for utility mode operations.
@@ -384,7 +385,7 @@ AORelCreateHashEntry(Oid relid)
 	aoHashEntry->head_rel_segfile.xid = InvalidTransactionId;
 	aoHashEntry->head_rel_segfile.latestWriteXid = InvalidTransactionId;
 	aoHashEntry->head_rel_segfile.isfull = true;
-    aoHashEntry->head_rel_segfile.needCheck = true;
+  aoHashEntry->head_rel_segfile.needCheck = true;
 	aoHashEntry->head_rel_segfile.tupcount = 0;
 	aoHashEntry->head_rel_segfile.tupsadded = 0;
 	aoHashEntry->max_seg_no = 0;
@@ -720,7 +721,7 @@ AppendOnlyRelHashNew(Oid relid, bool *exists)
 
 					/* remove this unused entry */
 					/* TODO: remove the LRU entry, not just any unused one */
-                    AORelRemoveHashEntry(hentry->relid, false);
+          AORelRemoveHashEntry(hentry->relid, false);
 					hash_seq_term(&status);
 
 					/* we now have room for a new entry, create it */
@@ -1117,7 +1118,7 @@ List *SetSegnoForWrite(List *existing_segnos, Oid relid, int segment_num,
                     {
                         LWLockRelease(AOSegFileLock);
                         ereport(ERROR, (errmsg("cannot open more than %d append-only table
segment files cocurrently",
-                                        MAX_AOREL_SEGFILE_STATUS)));
+                                        MaxAORelSegFileStatus)));
                     }
                     AOSegfileStatusPool[new_status].segno = ++aoentry->max_seg_no;
                     AOSegfileStatusPool[new_status].next = aoentry->head_rel_segfile.next;
@@ -1676,7 +1677,7 @@ AORelGetSegfileStatus(void)
 {
 	int result;
 
-	if (AppendOnlyWriter->num_existing_segfilestatus + 1 > MAX_AOREL_SEGFILE_STATUS)
+	if (AppendOnlyWriter->num_existing_segfilestatus + 1 > MaxAORelSegFileStatus)
 	{
 		return NEXT_END_OF_LIST;
 	}
@@ -1692,7 +1693,7 @@ AORelGetSegfileStatus(void)
 static void
 AORelPutSegfileStatus(int id)
 {
-	Assert((id >= 0) && (id < MAX_AOREL_SEGFILE_STATUS));
+	Assert((id >= 0) && (id < MaxAORelSegFileStatus));
 	Assert(AOSegfileStatusPool[id].id == id);
 
 	AOSegfileStatusPool[id].next = AppendOnlyWriter->head_free_segfilestatus;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/be4aabb7/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 4a96fe3..5c1ef45 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -30,6 +30,7 @@
 
 #include <signal.h>
 
+#include "access/appendonlywriter.h"
 #include "access/gin.h"
 #include "access/transam.h"
 #include "access/aosegfiles.h"
@@ -5437,6 +5438,15 @@ static struct config_int ConfigureNamesInt[] =
 	},
 
 	{
+	  {"max_appendonly_segfiles", PGC_POSTMASTER, APPENDONLY_TABLES,
+	    gettext_noop("Maximum number of different (unrelated) appendonly table segment files
that can be opened concurrently."),
+	    NULL
+	  },
+	  &MaxAORelSegFileStatus,
+	  262144, 2048, INT_MAX, NULL, NULL
+	},
+
+	{
 		{"test_appendonly_version_default", PGC_USERSET, APPENDONLY_TABLES,
 		 gettext_noop("Align append-only blocks to 64 bits."),
 		 NULL,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/be4aabb7/src/include/access/appendonlywriter.h
----------------------------------------------------------------------
diff --git a/src/include/access/appendonlywriter.h b/src/include/access/appendonlywriter.h
index 524d82e..bab8739 100755
--- a/src/include/access/appendonlywriter.h
+++ b/src/include/access/appendonlywriter.h
@@ -21,12 +21,6 @@
 #include "storage/lock.h"
 #include "tcop/dest.h"
 
-/*
- * Maximum concurrent number of write segment files for AO relation.
- * TODO: may want to make this a guc instead (can only be set at gpinit time).
- */
-#define MAX_AOREL_SEGFILE_STATUS (2048 * 128)
-
 #define NEXT_END_OF_LIST (-1)
 
 /*
@@ -50,7 +44,7 @@
  * GUC variables
  */
 extern int	MaxAppendOnlyTables;	/* Max # of concurrently used AO rels */
-
+extern int MaxAORelSegFileStatus; /* Max # of concurrently used AO segfiles */
 /*
  * Describes the status of a particular file segment number accross an entire
  * AO relation. 


Mime
View raw message