hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From iw...@apache.org
Subject [1/4] incubator-hawq git commit: HAWQ-265. Change metadata share memory flush strategy to prevent out of share memory problem when create too many hdfs_file.
Date Thu, 24 Dec 2015 02:44:13 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master db1faf9d9 -> 754e11ddb


HAWQ-265. Change metadata share memory flush strategy to prevent out of share memory problem
when create too many hdfs_file.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/bf648450
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/bf648450
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/bf648450

Branch: refs/heads/master
Commit: bf648450b06db05335aa449d2ec239301a4bf436
Parents: 8239cb9
Author: stanlyxiang <stanly.sxiang@gmail.com>
Authored: Mon Dec 21 10:52:15 2015 +0800
Committer: stanlyxiang <stanly.sxiang@gmail.com>
Committed: Mon Dec 21 10:52:15 2015 +0800

----------------------------------------------------------------------
 src/backend/cdb/cdbmetadatacache.c         |  5 ++---
 src/backend/cdb/cdbmetadatacache_process.c | 14 ++++++++++++--
 src/backend/cdb/cdbvars.c                  |  4 ++++
 src/backend/utils/misc/guc.c               | 25 +++++++++++++++++++++++++
 src/include/cdb/cdbvars.h                  |  4 ++++
 5 files changed, 47 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bf648450/src/backend/cdb/cdbmetadatacache.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbmetadatacache.c b/src/backend/cdb/cdbmetadatacache.c
index cef839f..f4c2369 100644
--- a/src/backend/cdb/cdbmetadatacache.c
+++ b/src/backend/cdb/cdbmetadatacache.c
@@ -65,7 +65,6 @@
 #include "fmgr.h"
 #include "utils/builtins.h"
 
-#define MAX_HDFS_FILE_NUM               (2 << 15)       // 1PB / 32G = 32K, 2^15
 #define MAX_HDFS_HOST_NUM               1024
 #define MAX_BLOCK_INFO_LEN              128
 #define BLOCK_INFO_BIT_NUM              16
@@ -184,7 +183,7 @@ MetadataCache_ShmemSize(void)
 {
     Size size;
     
-    size = hash_estimate_size((Size)MAX_HDFS_FILE_NUM, sizeof(MetadataCacheEntry));
+    size = hash_estimate_size((Size)metadata_cache_max_hdfs_file_num, sizeof(MetadataCacheEntry));
 
     size = add_size(size, hash_estimate_size((Size)MAX_HDFS_HOST_NUM, sizeof(BlockInfoEntry))
* METADATA_BLOCK_INFO_TYPE_NUM);
 
@@ -267,7 +266,7 @@ MetadataCacheHashTableInit(void)
     info.hash = tag_hash;
     hash_flags = (HASH_ELEM | HASH_FUNCTION);
 
-    MetadataCache = ShmemInitHash("Metadata Cache", MAX_HDFS_FILE_NUM, MAX_HDFS_FILE_NUM,
&info, hash_flags);
+    MetadataCache = ShmemInitHash("Metadata Cache", metadata_cache_max_hdfs_file_num, metadata_cache_max_hdfs_file_num,
&info, hash_flags);
     if (NULL == MetadataCache)
     {
         return false;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bf648450/src/backend/cdb/cdbmetadatacache_process.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbmetadatacache_process.c b/src/backend/cdb/cdbmetadatacache_process.c
index df4bbac..f289e3b 100644
--- a/src/backend/cdb/cdbmetadatacache_process.c
+++ b/src/backend/cdb/cdbmetadatacache_process.c
@@ -410,11 +410,18 @@ ProcessMetadataCacheCheck()
     HdfsFileInfo *file_info;
 
     double free_block_ratio = (FREE_BLOCK_NUM * 1.0) / metadata_cache_block_capacity;
+    long cache_entry_num = hash_get_num_entries(MetadataCache);
+    double cache_entry_ratio = (cache_entry_num * 1.0) / metadata_cache_max_hdfs_file_num;
 
     elog(DEBUG1, "[MetadataCache] ProcessMetadataCacheCheck free_block_ratio:%f", free_block_ratio);
 
-    if (free_block_ratio < metadata_cache_free_block_max_ratio)
+    if (free_block_ratio < metadata_cache_free_block_max_ratio || cache_entry_ratio >
metadata_cache_flush_ratio)
     {
+    	if(cache_entry_num == metadata_cache_max_hdfs_file_num)
+    	{
+    		elog(WARNING, "[MetadataCache] ProcessMetadataCacheCheck : Metadata cache is full.");
+    	}
+    	elog(DEBUG1, "[MetadataCache] ProcessMetadataCacheCheck cache_entry_ratio:%f", cache_entry_ratio);
         if (NULL == MetadataCacheLRUList) 
         {
             GenerateMetadataCacheLRUList();
@@ -436,7 +443,10 @@ ProcessMetadataCacheCheck()
             DestroyHdfsFileInfo(file_info);
             total_remove_files++;
 
-            if (((FREE_BLOCK_NUM * 1.0) / metadata_cache_block_capacity) >= metadata_cache_free_block_normal_ratio)
+            double cache_entry_ratio = hash_get_num_entries(MetadataCache) / metadata_cache_max_hdfs_file_num;
+
+            if (((FREE_BLOCK_NUM * 1.0) / metadata_cache_block_capacity) >= metadata_cache_free_block_normal_ratio
+            		&& cache_entry_ratio < metadata_cache_reduce_ratio)
             {
                 break;
             }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bf648450/src/backend/cdb/cdbvars.c
----------------------------------------------------------------------
diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c
index 76ee65c..e9021ee 100644
--- a/src/backend/cdb/cdbvars.c
+++ b/src/backend/cdb/cdbvars.c
@@ -278,6 +278,10 @@ int metadata_cache_refresh_max_num;
 double metadata_cache_free_block_max_ratio;
 double metadata_cache_free_block_normal_ratio;
 
+int metadata_cache_max_hdfs_file_num;
+double metadata_cache_flush_ratio;
+double metadata_cache_reduce_ratio;
+
 char *metadata_cache_testfile;
 bool debug_fake_datalocality;
 bool datalocality_remedy_enable;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bf648450/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 2880067..f1e5f8a 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6533,6 +6533,15 @@ static struct config_int ConfigureNamesInt[] =
 		&metadata_cache_refresh_max_num,
 		1000, 1, 10000, NULL, NULL
 	},
+	{
+		{
+			"hawq_metadata_cache_max_hdfs_file_num", PGC_POSTMASTER, DEVELOPER_OPTIONS,
+				gettext_noop("max hdfs file num in metadata."),
+				NULL
+		},
+		&metadata_cache_max_hdfs_file_num,
+		524288, 32768, 8388608, NULL, NULL
+	},
 
 	/* End-of-list marker */
 	{
@@ -6559,6 +6568,22 @@ static struct config_real ConfigureNamesReal[] =
 		&metadata_cache_free_block_normal_ratio,
 		0.2, 0.05, 1.0, NULL, NULL
 	},
+	{
+		{"hawq_metadata_cache_flush_ratio", PGC_POSTMASTER, DEVELOPER_OPTIONS,
+				gettext_noop("metadata cache flush ratio."),
+				NULL
+		},
+		&metadata_cache_flush_ratio,
+		0.85, 0.50, 1.0, NULL, NULL
+	},
+	{
+		{"hawq_metadata_cache_reduce_ratio", PGC_POSTMASTER, DEVELOPER_OPTIONS,
+				gettext_noop("metadata cache reduce ratio."),
+				NULL
+		},
+		&metadata_cache_reduce_ratio,
+		0.7, 0.50, 1.0, NULL, NULL
+	},
 
 	{
 		{"seq_page_cost", PGC_USERSET, QUERY_TUNING_COST,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/bf648450/src/include/cdb/cdbvars.h
----------------------------------------------------------------------
diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h
index ff10a76..6526168 100644
--- a/src/include/cdb/cdbvars.h
+++ b/src/include/cdb/cdbvars.h
@@ -1137,6 +1137,10 @@ extern int metadata_cache_refresh_max_num;
 extern double metadata_cache_free_block_max_ratio;
 extern double metadata_cache_free_block_normal_ratio;
 
+extern int metadata_cache_max_hdfs_file_num;
+extern double metadata_cache_flush_ratio;
+extern double metadata_cache_reduce_ratio;
+
 extern char *metadata_cache_testfile;
 extern bool debug_fake_datalocality;
 extern bool datalocality_remedy_enable;


Mime
View raw message