hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject incubator-hawq git commit: HAWQ-780. Remove quicklz compression related code but keep related meta data in short term.
Date Wed, 08 Jun 2016 05:26:56 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 54fcf1461 -> d65ca5975


HAWQ-780. Remove quicklz compression related code but keep related meta data in short term.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/d65ca597
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/d65ca597
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/d65ca597

Branch: refs/heads/master
Commit: d65ca5975105fc4715507d1023d7861ff0d35c13
Parents: 54fcf14
Author: Paul Guo <paulguo@gmail.com>
Authored: Tue Jun 7 15:25:48 2016 +0800
Committer: rlei <rlei@pivotal.io>
Committed: Wed Jun 8 13:26:33 2016 +0800

----------------------------------------------------------------------
 configure                               | 16 ----------
 configure.in                            | 11 -------
 src/Makefile.global.in                  |  3 --
 src/all_src_files.txt                   |  4 ---
 src/backend/Makefile                    |  7 +----
 src/backend/access/common/reloptions.c  | 42 ++------------------------
 src/backend/catalog/Makefile            |  6 +---
 src/backend/storage/file/README_QUICKLZ | 36 -----------------------
 src/backend/utils/misc/guc.c            | 44 ----------------------------
 src/include/catalog/pg_appendonly.h     |  2 +-
 src/include/catalog/pg_compression.h    |  3 +-
 src/include/storage/gp_compress.h       | 13 --------
 src/include/utils/builtins.h            |  6 ----
 src/include/utils/guc.h                 |  4 ---
 14 files changed, 6 insertions(+), 191 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 20346e6..abb93e7 100755
--- a/configure
+++ b/configure
@@ -716,7 +716,6 @@ ELF_SYS
 EGREP
 GREP
 with_rt
-ADDON_DIR
 with_zlib
 with_system_tzdata
 with_libhdfs3
@@ -5733,7 +5732,6 @@ fi
 $as_echo "$with_java" >&6; }
 
 
-
 #
 # Optionally build R modules (PL/R)
 #
@@ -5766,15 +5764,6 @@ fi
 
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $with_r" >&5
 $as_echo "$with_r" >&6; }
-{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING:
-*** For license reason, we do not install pl/plr into hawq, instead please build the package
-*** yourself by running \"make -C src/pl plr_pkgbuild\" on the root workspace path.
-*** Then the package could be found under src/pl/build." >&5
-$as_echo "$as_me: WARNING:
-*** For license reason, we do not install pl/plr into hawq, instead please build the package
-*** yourself by running \"make -C src/pl plr_pkgbuild\" on the root workspace path.
-*** Then the package could be found under src/pl/build." >&2;}
-
 
 
 #
@@ -6294,11 +6283,6 @@ fi
 
 
 #
-# addon
-#
-
-
-#
 # Realtime library
 #
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index 684bc58..89f9b7e 100644
--- a/configure.in
+++ b/configure.in
@@ -698,20 +698,14 @@ PGAC_ARG_BOOL(with, java, no, [  --with-java             build Java
modules (PL/
 AC_MSG_RESULT([$with_java])
 AC_SUBST(with_java)
 
-
 #
 # Optionally build R modules (PL/R)
 #
 AC_MSG_CHECKING([whether to build R modules])
 PGAC_ARG_BOOL(with, r, no, [  --with-r                build R modules (PL/R)])
 AC_MSG_RESULT([$with_r])
-AC_MSG_WARN([
-*** For license reason, we do not install pl/plr into hawq, instead please build the package
-*** yourself by running "make -C src/pl plr_pkgbuild" on the root workspace path.
-*** Then the package could be found under src/pl/build.])
 AC_SUBST(with_r)
 
-
 #
 # Optionally build with pgcrypto.
 #
@@ -862,11 +856,6 @@ PGAC_ARG_BOOL(with, zlib, yes,
 AC_SUBST(with_zlib)
 
 #
-# addon
-#
-AC_SUBST(ADDON_DIR)
-
-#
 # Realtime library
 #
 PGAC_ARG_BOOL(with, rt, yes,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/Makefile.global.in
----------------------------------------------------------------------
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index dc09653..5388cf4 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -39,9 +39,6 @@ GP_MAJORVERSION = @GP_MAJORVERSION@
 # HAWQ version number
 HQ_MAJORVERSION = @HQ_MAJORVERSION@
 
-# addon
-ADDON_DIR = @ADDON_DIR@
-
 # Support for VPATH builds
 vpath_build = @vpath_build@
 abs_top_srcdir = @abs_top_srcdir@

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/all_src_files.txt
----------------------------------------------------------------------
diff --git a/src/all_src_files.txt b/src/all_src_files.txt
index f85326a..f9899dd 100644
--- a/src/all_src_files.txt
+++ b/src/all_src_files.txt
@@ -590,8 +590,6 @@ backend/storage/file/fd.c
 backend/storage/file/filesystem.c
 backend/storage/file/gp_compress.c
 backend/storage/file/pipe.c
-backend/storage/file/quicklz1.c
-backend/storage/file/quicklz3.c
 backend/storage/freespace/freespace.c
 backend/storage/ipc/ipc.c
 backend/storage/ipc/ipci.c
@@ -1355,8 +1353,6 @@ include/storage/pmsignal.h
 include/storage/pos.h
 include/storage/proc.h
 include/storage/procarray.h
-include/storage/quicklz1.h
-include/storage/quicklz3.h
 include/storage/relfilenode.h
 include/storage/s_lock.h
 include/storage/shmem.h

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 20143dd..8643775 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -14,14 +14,9 @@ subdir = src/backend
 top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 
-ifdef ADDON_DIR
-include $(ADDON_DIR)/src/Makefile
-endif
-
 SUBDIRS = access bootstrap catalog parser commands executor foreign \
 	lib libpq gp_libpq_fe main nodes optimizer port postmaster regex \
-	rewrite storage tcop utils resourcemanager $(top_builddir)/src/timezone cdb \
-	$(ADDON_SUBDIR)
+	rewrite storage tcop utils resourcemanager $(top_builddir)/src/timezone cdb
 
 include $(srcdir)/common.mk
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/backend/access/common/reloptions.c
----------------------------------------------------------------------
diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c
index 8822d01..a2a02e1 100644
--- a/src/backend/access/common/reloptions.c
+++ b/src/backend/access/common/reloptions.c
@@ -612,23 +612,6 @@ default_reloptions(Datum reloptions, bool validate, char relkind,
 				compresstype = pstrdup(defaultParquetCompressor);
 		}
 
-		if (compresstype && (pg_strcasecmp(compresstype, "quicklz") == 0) &&
-		    (compresslevel != 1))
-		{
-		  /* allow quicklz level 3 only if the debug guc is on */
-		  if (!(compresslevel == 3 && Test_enable_broken_quicklz3))
-		  {
-		    if (validate)
-		      ereport(ERROR,
-		          (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
-		           errmsg("compresslevel=%d is out of range for quicklz "
-		               "(should be 1)", compresslevel),
-		               errOmitLocation(true)));
-
-		      compresslevel = setDefaultCompressionLevel(compresstype);
-		  }
-		}
-
 		if (compresstype && (pg_strcasecmp(compresstype, "rle_type") == 0) &&
 			(compresslevel > 4))
 		{
@@ -1012,7 +995,7 @@ void validateAppendOnlyRelOptions(bool ao,
 	}
 
 	if (comptype &&
-		(pg_strcasecmp(comptype, "quicklz") == 0 ||
+		(pg_strcasecmp(comptype, "snappy") == 0 ||
 		 pg_strcasecmp(comptype, "zlib") == 0 ||
 		 pg_strcasecmp(comptype, "rle_type") == 0))
 	{
@@ -1025,15 +1008,6 @@ void validateAppendOnlyRelOptions(bool ao,
 							comptype)));			
 		}
 		
-		if ((colstore != RELSTORAGE_PARQUET) &&
-			pg_strcasecmp(comptype, "snappy") == 0)
-		{
-			ereport(ERROR,
-					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
-					 errmsg("%s can only be used with parquet relations",
-							comptype)));
-		}
-
 		if (comptype && (pg_strcasecmp(comptype, "snappy") != 0)&& complevel ==
0)
 			ereport(ERROR,
 					(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
@@ -1045,18 +1019,6 @@ void validateAppendOnlyRelOptions(bool ao,
 					 errmsg("compresslevel=%d is out of range (should be between 0 and 9)",
 							complevel)));
 
-		if (comptype && (pg_strcasecmp(comptype, "quicklz") == 0) &&
-			(complevel != 1))
-		{
-			/* allow quicklz level 3 only if the debug guc is on */
-			if (!(complevel == 3 && Test_enable_broken_quicklz3))
-			{
-				ereport(ERROR,
-						(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
-						 errmsg("compresslevel=%d is out of range for quicklz "
-								 "(should be 1)", complevel)));
-			}
-		}
 		if (comptype && (pg_strcasecmp(comptype, "rle_type") == 0) &&
 			(complevel > 4))
 		{
@@ -1098,7 +1060,7 @@ void validateAppendOnlyRelOptions(bool ao,
 
 /*
  * if no compressor type was specified, we set to no compression (level 0)
- * otherwise default for both zlib and quicklz is level 1. RLE_TYPE does
+ * otherwise default for both zlib is level 1. RLE_TYPE does
  * not have a compression level.
  */
 static int setDefaultCompressionLevel(char* compresstype)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/backend/catalog/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
index b453d38..68bfa2b 100644
--- a/src/backend/catalog/Makefile
+++ b/src/backend/catalog/Makefile
@@ -10,17 +10,13 @@ subdir = src/backend/catalog
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-ifndef ADDON_DIR
-QUICKLZ_COMPRESSION = quicklz_compression.o
-endif
-
 OBJS = catalog.o dependency.o heap.o index.o indexing.o namespace.o aclchk.o \
        gp_global_sequence.o gp_persistent.o \
        pg_aggregate.o pg_appendonly.o pg_constraint.o pg_conversion.o pg_depend.o \
        pg_exttable.o pg_extprotocol.o pg_filesystem.o pg_largeobject.o pg_namespace.o \
        pg_operator.o pg_proc.o pg_proc_callback.o pg_shdepend.o \
        pg_type.o toasting.o aoseg.o \
-	pg_attribute_encoding.o pg_compression.o $(QUICKLZ_COMPRESSION)
+       pg_attribute_encoding.o pg_compression.o quicklz_compression.o
 
 SUBDIRS = caql core external
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/backend/storage/file/README_QUICKLZ
----------------------------------------------------------------------
diff --git a/src/backend/storage/file/README_QUICKLZ b/src/backend/storage/file/README_QUICKLZ
deleted file mode 100644
index ff27814..0000000
--- a/src/backend/storage/file/README_QUICKLZ
+++ /dev/null
@@ -1,36 +0,0 @@
-
-QUICKLZ COMPRESSION LIBRARIES
-==============================
-
-QuickLZ 1.4.0 comes out of the box as a set of two files:
- - quicklz.c
- - quicklz.h
- 
-The library file includes a set of pre-compiled options, one of which is
-of interest to us: QLZ_COMPRESSION_LEVEL. Level 1 gives the fastest compression 
-speed while level 3 gives the fastest decompression speed. 
-
-Quicklz state that: "Because of performance reasons these settings cannot be 
-specified dynamically at runtime. They must be specified in the beginning of the 
-quicklz.h file whereafter the library must be compiled.�
-
-Therefore, in order to be able to support dynamically selecting a compression 
-level (values 1 or 3) we decided to create 2 sets of files that are pre compiled
-one for compression level 1 and another for compression level 3. we changed the
-API in each of them to include only values and function calls relevant to the
-specific compression level.
-
-compression level 1 must use the functions and constants from here:
- - quicklz1.c
- - quicklz1.h
-
-compression level 3 must use the functions and constants from here:
- - quicklz3.c
- - quicklz3.h
-
-Practically this means that there are 3 types of compressors in GPDB at this
-moment:
-
- 1) zlib (with levels 1 - 9)
- 2) quicklz with level 1
- 3) quicklz with level 3

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/backend/utils/misc/guc.c
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 748705f..eb01c35 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -320,7 +320,6 @@ bool		Test_print_direct_dispatch_info = false;
 bool		gp_permit_persistent_metadata_update = false;
 bool		gp_permit_relation_node_change = false;
 bool 		Test_checksum_override = false;
-bool		Test_enable_broken_quicklz3 = false;
 int         Test_compresslevel_override = 0;
 int			Test_blocksize_override = 0;
 int			Test_safefswritesize_override = 0;
@@ -328,9 +327,6 @@ bool        Master_mirroring_administrator_disable = false;
 bool		gp_appendonly_verify_block_checksums = false;
 bool 		gp_appendonly_verify_write_block = false;
 bool		gp_heap_require_relhasoids_match = true;
-bool		Debug_appendonly_rezero_quicklz_compress_scratch = false;
-bool		Debug_appendonly_rezero_quicklz_decompress_scratch = false;
-bool		Debug_appendonly_guard_end_quicklz_scratch = false;
 bool 		gp_local_distributed_cache_stats = false;
 bool		Debug_xlog_insert_print = false;
 bool		Debug_persistent_print = false;
@@ -2153,36 +2149,6 @@ static struct config_bool ConfigureNamesBool[] =
 	},
 
 	{
-		{"debug_appendonly_rezero_quicklz_compress_scratch", PGC_USERSET, DEVELOPER_OPTIONS,
-		 gettext_noop("Zero the QuickLZ scratch buffer before each append-only block that is being
compressed."),
-		 NULL,
-		 GUC_NOT_IN_SAMPLE | GUC_NO_SHOW_ALL
-		},
-		&Debug_appendonly_rezero_quicklz_compress_scratch,
-		false, NULL, NULL
-	},
-
-	{
-		{"debug_appendonly_guard_end_quicklz_scratch", PGC_USERSET, DEVELOPER_OPTIONS,
-		 gettext_noop("Put a guard area of all zeroes after the QuickLZ scratch buffers and verify
it is still zero before and after compress and decompress operations."),
-		 NULL,
-		 GUC_NOT_IN_SAMPLE | GUC_NO_SHOW_ALL
-		},
-		&Debug_appendonly_guard_end_quicklz_scratch,
-		false, NULL, NULL
-	},
-
-	{
-		{"debug_appendonly_rezero_quicklz_decompress_scratch", PGC_USERSET, DEVELOPER_OPTIONS,
-		 gettext_noop("Zero the QuickLZ scratch buffer before each append-only block that is being
decompressed."),
-		 NULL,
-		 GUC_NOT_IN_SAMPLE | GUC_NO_SHOW_ALL
-		},
-		&Debug_appendonly_rezero_quicklz_decompress_scratch,
-		false, NULL, NULL
-	},
-
-	{
 		{"gp_cost_hashjoin_chainwalk", PGC_USERSET, QUERY_TUNING_COST,
 			gettext_noop("Enable the cost for walking the chain in the hash join"),
 			NULL,
@@ -2889,16 +2855,6 @@ static struct config_bool ConfigureNamesBool[] =
 	},
 
 	{
-		{"test_enable_broken_quicklz3", PGC_SUSET, DEVELOPER_OPTIONS,
-			gettext_noop("For testing purposes, allow using quicklz level 3 compression."),
-			NULL,
-			GUC_SUPERUSER_ONLY |  GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE
-		},
-		&Test_enable_broken_quicklz3,
-		false, NULL, NULL
-	},
-
-	{
 		{"master_mirroring_administrator_disable", PGC_POSTMASTER, GP_ARRAY_CONFIGURATION,
 			gettext_noop("Used by gpstart to indicate the standby master was not started by the administrator."),
 			NULL,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/include/catalog/pg_appendonly.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_appendonly.h b/src/include/catalog/pg_appendonly.h
index 5f0ef04..1f8142e 100644
--- a/src/include/catalog/pg_appendonly.h
+++ b/src/include/catalog/pg_appendonly.h
@@ -75,7 +75,7 @@ CATALOG(pg_appendonly,6105) BKI_WITHOUT_OIDS
 	int2			majorversion;		/* major version indicating what's stored in this table  */
 	int2			minorversion;		/* minor version indicating what's stored in this table  */
 	bool			checksum;			/* true if checksum is stored with data and checked */
-	text			compresstype;		/* the compressor used (zlib, or or snappy, quicklz) */
+	text			compresstype;		/* the compressor used (zlib or snappy) */
     bool            columnstore;        /* true if co or parquet table, false if ao table*/
     Oid             segrelid;           /* OID of aoseg table; 0 if none */
     Oid             segidxid;           /* if aoseg table, OID of segno index */

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/include/catalog/pg_compression.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_compression.h b/src/include/catalog/pg_compression.h
index 8746db9..eb39ba1 100644
--- a/src/include/catalog/pg_compression.h
+++ b/src/include/catalog/pg_compression.h
@@ -140,8 +140,7 @@ typedef struct CompressionState
 	/*
 	 * Allows a constructor to tell the calling level the maximum storage
 	 * required for input of the given size. Different algorithms need
-	 * different maximum buffers. For example, quicklz's compression
-	 * algorithm needs at most input size + 400 bytes.
+	 * different maximum buffers.
 	 */
 	size_t (*desired_sz)(size_t input);
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/include/storage/gp_compress.h
----------------------------------------------------------------------
diff --git a/src/include/storage/gp_compress.h b/src/include/storage/gp_compress.h
index df345cd..61ff04d 100644
--- a/src/include/storage/gp_compress.h
+++ b/src/include/storage/gp_compress.h
@@ -33,19 +33,6 @@
 #include <zlib.h>
 #endif
 
-
-/*
- *	Represents the compressor type used for an AO table.
- *	Note that we treat quicklz with compression levels 1 and 3 as
- *	two separate types of compressors, this is because we have to
- *	compile them separately and have a separate API for each. Zlib
- *	compression will have a compression level attached to it, checked
- *	separately.
- */
-
-// UNDONE: Is there a QLZ DEFINE for this?
-#define MAX_OVERRUN_QUICKLZ 400
-
 extern int gp_trycompress_new(
 		 uint8			*sourceData,
 		 int32			 sourceLen,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/include/utils/builtins.h
----------------------------------------------------------------------
diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h
index 4dca6de..0b7e30d 100644
--- a/src/include/utils/builtins.h
+++ b/src/include/utils/builtins.h
@@ -1158,11 +1158,6 @@ extern Datum pg_partition_oid_transfn(PG_FUNCTION_ARGS);
 extern Datum pg_partition_oid_finalfn(PG_FUNCTION_ARGS);
 
 /* storage/compress.c */
-extern Datum quicklz_constructor(PG_FUNCTION_ARGS);
-extern Datum quicklz_destructor(PG_FUNCTION_ARGS);
-extern Datum quicklz_compress(PG_FUNCTION_ARGS);
-extern Datum quicklz_decompress(PG_FUNCTION_ARGS);
-extern Datum quicklz_validator(PG_FUNCTION_ARGS);
 
 extern Datum snappy_constructor(PG_FUNCTION_ARGS);
 extern Datum snappy_destructor(PG_FUNCTION_ARGS);
@@ -1196,7 +1191,6 @@ extern Datum dummy_compression_validator(PG_FUNCTION_ARGS);
 
 extern Datum gp_compressor(PG_FUNCTION_ARGS);
 extern Datum gp_decompressor(PG_FUNCTION_ARGS);
-extern Datum test_quicklz_compression(PG_FUNCTION_ARGS);
 
 /* percentile.c */
 extern Datum percentile_cont_trans(PG_FUNCTION_ARGS);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/d65ca597/src/include/utils/guc.h
----------------------------------------------------------------------
diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h
index f514de4..2e4e6eb 100644
--- a/src/include/utils/guc.h
+++ b/src/include/utils/guc.h
@@ -166,7 +166,6 @@ extern bool gp_persistent_statechange_suppress_error;
 extern bool Debug_bitmap_print_insert;
 extern bool Test_appendonly_override;
 extern bool	Gohdb_appendonly_override;
-extern bool	Test_enable_broken_quicklz3;
 extern bool	gp_permit_persistent_metadata_update;
 extern bool gp_permit_relation_node_change;
 extern bool Test_checksum_override;
@@ -178,9 +177,6 @@ extern bool gp_local_distributed_cache_stats;
 extern bool gp_appendonly_verify_block_checksums;
 extern bool gp_appendonly_verify_write_block;
 extern bool gp_heap_require_relhasoids_match;
-extern bool	Debug_appendonly_rezero_quicklz_compress_scratch;
-extern bool	Debug_appendonly_rezero_quicklz_decompress_scratch;
-extern bool	Debug_appendonly_guard_end_quicklz_scratch;
 extern bool	Debug_xlog_insert_print;
 extern bool	Debug_persistent_print;
 extern int	Debug_persistent_print_level;


Mime
View raw message