hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h...@apache.org
Subject incubator-hawq git commit: HAWQ-300. Fix out of memory during loading large volume data on large cluster with YARN mode
Date Wed, 06 Jan 2016 07:19:32 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 1e535bd4d -> 7e0ff4193


HAWQ-300. Fix out of memory during loading large volume data on large cluster with YARN mode


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7e0ff419
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7e0ff419
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7e0ff419

Branch: refs/heads/master
Commit: 7e0ff41933e4efc58918656dfdedae496432a3f0
Parents: 1e535bd
Author: Ruilong Huo <rhuo@pivotal.io>
Authored: Tue Jan 5 22:15:54 2016 -0800
Committer: Ruilong Huo <rhuo@pivotal.io>
Committed: Tue Jan 5 22:15:54 2016 -0800

----------------------------------------------------------------------
 src/backend/executor/execMain.c                   | 18 +++++++++++++++++-
 src/backend/utils/init/globals.c                  |  1 +
 src/backend/utils/misc/guc.c                      | 10 ++++++++++
 src/include/miscadmin.h                           |  1 +
 src/test/regress/expected/parquet_compression.out |  2 +-
 src/test/regress/sql/parquet_compression.sql      |  2 +-
 6 files changed, 31 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/backend/executor/execMain.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index f5a2038..831e557 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -109,6 +109,8 @@
 #include "cdb/cdbquerycontextdispatching.h"
 #include "optimizer/prep.h"
 
+#include "resourcemanager/dynrm.h"
+
 extern bool		filesystem_support_truncate;
 
 typedef struct evalPlanQual
@@ -440,8 +442,22 @@ ExecutorStart(QueryDesc *queryDesc, int eflags)
              * resource queues.
              */
             if (queryDesc->plannedstmt->query_mem > 0) {
+
+                uint64 memAvailableBytes = 0;
+
+                /* With resouce manager in NONE mode, we assign memory quota for operators
normally */
+                if ( strcasecmp(rm_global_rm_type, HAWQDRM_CONFFILE_SVRTYPE_VAL_NONE) ==
0 )
+                {
+                    memAvailableBytes = queryDesc->plannedstmt->query_mem;
+                }
+                /* With resouce manager in YARN/MESOS mode, we assign memory quota for operators
conservatively */
+                else
+                {
+                    memAvailableBytes = queryDesc->plannedstmt->query_mem * hawq_re_memory_quota_allocation_ratio;
+                }
+
                 PolicyEagerFreeAssignOperatorMemoryKB(queryDesc->plannedstmt,
-                                                      queryDesc->plannedstmt->query_mem);
+                                                      memAvailableBytes);
 
             }
         }

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/backend/utils/init/globals.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/init/globals.c b/src/backend/utils/init/globals.c
index 3ea1428..951b23d 100644
--- a/src/backend/utils/init/globals.c
+++ b/src/backend/utils/init/globals.c
@@ -150,4 +150,5 @@ int hawq_re_memory_overcommit_max = 8192;
 #else
 int hawq_re_memory_overcommit_max = 8192;
 #endif
+double hawq_re_memory_quota_allocation_ratio = 0.5;
 int gp_vmem_protect_gang_cache_limit = 500;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index becc9c9..3f591d4 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -6956,6 +6956,16 @@ static struct config_real ConfigureNamesReal[] =
 		3.0, 1.0, 100.0, NULL, NULL
 	},
 
+	{
+		{"hawq_re_memory_quota_allocation_ratio", PGC_USERSET, RESOURCES_MEM,
+			gettext_noop("Sets the ratio for memory quota allocation during query optimization."),
+			NULL,
+			GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE
+		},
+		&hawq_re_memory_quota_allocation_ratio,
+		0.5, 0.0, 1.0, NULL, NULL
+	},
+
 /* End-of-list marker */
 	{
 		{NULL, 0, 0, NULL, NULL}, NULL, 0.0, 0.0, 0.0, NULL, NULL

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/include/miscadmin.h
----------------------------------------------------------------------
diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h
index 8a90b5e..ce362e4 100644
--- a/src/include/miscadmin.h
+++ b/src/include/miscadmin.h
@@ -314,6 +314,7 @@ extern int	VacuumCostBalance;
 extern bool VacuumCostActive;
 
 extern int hawq_re_memory_overcommit_max;
+extern double hawq_re_memory_quota_allocation_ratio;
 extern int gp_vmem_protect_gang_cache_limit;
 
 /* in tcop/postgres.c */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/test/regress/expected/parquet_compression.out
----------------------------------------------------------------------
diff --git a/src/test/regress/expected/parquet_compression.out b/src/test/regress/expected/parquet_compression.out
index 5c01573..34feee3 100644
--- a/src/test/regress/expected/parquet_compression.out
+++ b/src/test/regress/expected/parquet_compression.out
@@ -26,7 +26,7 @@ drop table parquet_snappy_part_unc;
 ERROR:  table "parquet_snappy_part_unc" does not exist
 drop table parquet_gzip_2;
 ERROR:  table "parquet_gzip_2" does not exist
-alter resource queue pg_default with ( vseg_resource_quota='mem:2gb');
+alter resource queue pg_default with ( vseg_resource_quota='mem:4gb');
 --set statement_mem='1999MB';
 --end_ignore
 --Datatypes covered: text,bytea,varchar,bit varying

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e0ff419/src/test/regress/sql/parquet_compression.sql
----------------------------------------------------------------------
diff --git a/src/test/regress/sql/parquet_compression.sql b/src/test/regress/sql/parquet_compression.sql
index 7a61d33..6e6bf3d 100644
--- a/src/test/regress/sql/parquet_compression.sql
+++ b/src/test/regress/sql/parquet_compression.sql
@@ -19,7 +19,7 @@ drop table parquet_snappy_part;
 drop table parquet_snappy_part_unc;
 drop table parquet_gzip_2;
 
-alter resource queue pg_default with ( vseg_resource_quota='mem:2gb');
+alter resource queue pg_default with ( vseg_resource_quota='mem:4gb');
 --set statement_mem='1999MB';
 --end_ignore
 


Mime
View raw message