hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject incubator-hawq git commit: HAWQ-72. Introduce ADDON_DIR macro in configure/makefile to allow custimizatin
Date Fri, 23 Oct 2015 07:53:44 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master b10897824 -> fdd2ae17b


HAWQ-72. Introduce ADDON_DIR macro in configure/makefile to allow custimizatin


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/fdd2ae17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/fdd2ae17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/fdd2ae17

Branch: refs/heads/master
Commit: fdd2ae17b4e46d1a50fb8c8a0953d681f226611a
Parents: b108978
Author: Lirong Jian <jianlirong@gmail.com>
Authored: Fri Oct 23 11:04:05 2015 +0800
Committer: Lirong Jian <jianlirong@gmail.com>
Committed: Fri Oct 23 11:04:26 2015 +0800

----------------------------------------------------------------------
 configure                                    |   6 +
 configure.in                                 |   5 +
 src/Makefile.global.in                       |   3 +
 src/backend/Makefile                         |   7 +-
 src/backend/access/appendonly/appendonlyam.c |   5 -
 src/backend/access/common/reloptions.c       |   5 +-
 src/backend/catalog/Makefile                 |   7 +-
 src/backend/catalog/pg_compression.c         | 210 ----------------------
 src/backend/catalog/quicklz_compression.c    |  58 ++++++
 src/include/catalog/pg_compression.h         |   2 +
 src/include/catalog/pg_proc.h                |  20 +++
 src/include/storage/gp_compress.h            |   5 -
 12 files changed, 107 insertions(+), 226 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 8dea6e1..309039f 100755
--- a/configure
+++ b/configure
@@ -714,6 +714,7 @@ ELF_SYS
 EGREP
 GREP
 with_rt
+ADDON_DIR
 with_zlib
 with_system_tzdata
 with_openssl
@@ -5986,6 +5987,11 @@ fi
 
 
 #
+# addon
+#
+
+
+#
 # Realtime library
 #
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index 8c7ae10..fbf6552 100644
--- a/configure.in
+++ b/configure.in
@@ -805,6 +805,11 @@ PGAC_ARG_BOOL(with, zlib, yes,
 AC_SUBST(with_zlib)
 
 #
+# addon
+#
+AC_SUBST(ADDON_DIR)
+
+#
 # Realtime library
 #
 PGAC_ARG_BOOL(with, rt, yes,

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/Makefile.global.in
----------------------------------------------------------------------
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index a619d74..1e569d1 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -37,6 +37,9 @@ GP_MAJORVERSION = @GP_MAJORVERSION@
 # HAWQ version number
 HQ_MAJORVERSION = @HQ_MAJORVERSION@
 
+# addon
+ADDON_DIR = @ADDON_DIR@
+
 # Support for VPATH builds
 vpath_build = @vpath_build@
 abs_top_srcdir = @abs_top_srcdir@

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/Makefile b/src/backend/Makefile
index 44f13a2..28bde84 100644
--- a/src/backend/Makefile
+++ b/src/backend/Makefile
@@ -14,9 +14,14 @@ subdir = src/backend
 top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 
+ifdef ADDON_DIR
+include $(ADDON_DIR)/src/Makefile
+endif
+
 SUBDIRS = access bootstrap catalog parser commands executor foreign \
 	lib libpq gp_libpq_fe main nodes optimizer port postmaster regex \
-	rewrite storage tcop utils resourcemanager $(top_builddir)/src/timezone cdb
+	rewrite storage tcop utils resourcemanager $(top_builddir)/src/timezone cdb \
+	$(ADDON_SUBDIR)
 
 include $(srcdir)/common.mk
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/access/appendonly/appendonlyam.c
----------------------------------------------------------------------
diff --git a/src/backend/access/appendonly/appendonlyam.c b/src/backend/access/appendonly/appendonlyam.c
index 45a0035..d08e82f 100755
--- a/src/backend/access/appendonly/appendonlyam.c
+++ b/src/backend/access/appendonly/appendonlyam.c
@@ -75,11 +75,6 @@
 #include "utils/builtins.h"
 #include "utils/guc.h"
 
-#ifdef HAVE_QUICKLZ
-#include <quicklz1.h>
-#include <quicklz3.h>
-#endif
-
 #ifdef HAVE_LIBZ
 #include <zlib.h>
 #endif

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/access/common/reloptions.c
----------------------------------------------------------------------
diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c
index 679cc93..38b404a 100644
--- a/src/backend/access/common/reloptions.c
+++ b/src/backend/access/common/reloptions.c
@@ -517,7 +517,6 @@ default_reloptions(Datum reloptions, bool validate, char relkind,
 
 		if (!compresstype_is_valid(compresstype))
 		{
-#ifndef HAVE_QUICKLZ
 		  if (strcmp(compresstype, "quicklz") == 0)
 		  {
 		    ereport(ERROR,
@@ -526,7 +525,6 @@ default_reloptions(Datum reloptions, bool validate, char relkind,
 		         errOmitLocation(true)));
 		  }
 		  else
-#endif
 		  {
 		    ereport(ERROR,
 		        (errcode(ERRCODE_UNDEFINED_OBJECT),
@@ -615,7 +613,6 @@ default_reloptions(Datum reloptions, bool validate, char relkind,
 				compresstype = pstrdup(defaultParquetCompressor);
 		}
 
-#ifdef HAVE_QUICKLZ
 		if (compresstype && (pg_strcasecmp(compresstype, "quicklz") == 0) &&
 		    (compresslevel != 1))
 		{
@@ -632,7 +629,7 @@ default_reloptions(Datum reloptions, bool validate, char relkind,
 		      compresslevel = setDefaultCompressionLevel(compresstype);
 		  }
 		}
-#endif
+
 		if (compresstype && (pg_strcasecmp(compresstype, "rle_type") == 0) &&
 			(compresslevel > 4))
 		{

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/catalog/Makefile
----------------------------------------------------------------------
diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile
index 342a6c6..015a3ab 100644
--- a/src/backend/catalog/Makefile
+++ b/src/backend/catalog/Makefile
@@ -10,13 +10,18 @@ subdir = src/backend/catalog
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+ifndef ADDON_DIR
+QUICKLZ_COMPRESSION = quicklz_compression.o
+endif
+
 OBJS = catalog.o dependency.o heap.o index.o indexing.o namespace.o aclchk.o \
        gp_global_sequence.o gp_persistent.o \
        pg_aggregate.o pg_appendonly.o pg_constraint.o pg_conversion.o pg_depend.o \
        pg_exttable.o pg_extprotocol.o pg_filesystem.o pg_largeobject.o pg_namespace.o \
        pg_operator.o pg_proc.o pg_proc_callback.o pg_shdepend.o \
        pg_type.o toasting.o aoseg.o aoblkdir.o gp_fastsequence.o \
-	   pg_attribute_encoding.o pg_compression.o
+	pg_attribute_encoding.o pg_compression.o $(QUICKLZ_COMPRESSION)
+
 SUBDIRS = caql core hcatalog
 
 BKIFILES = postgres.bki postgres.description postgres.shdescription

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/catalog/pg_compression.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/pg_compression.c b/src/backend/catalog/pg_compression.c
index c340ee8..e9714db 100644
--- a/src/backend/catalog/pg_compression.c
+++ b/src/backend/catalog/pg_compression.c
@@ -37,33 +37,10 @@
 #include "utils/relcache.h"
 #include "utils/syscache.h"
 
-#ifdef HAVE_QUICKLZ
-#include <quicklz1.h>
-#include <quicklz3.h>
-#endif
-
 /* names we expect to see in ENCODING clauses */
 char *storage_directive_names[] = {"compresstype", "compresslevel",
 								   "blocksize", NULL};
 
-#ifdef HAVE_QUICKLZ
-/* Internal state for quicklz */
-typedef struct quicklz_state
-{
-	void *scratch;
-
-	int level;
-	bool compress;
-
-	/*
-	 * The actual algorithms. Allows us to handle quicklz1 and quicklz3
-	 * conveniently.
-	 */
-	size_t (*compress_fn)(int level, const void *, char *, size_t, void *);
-	size_t (*decompress_fn)(int level, const char *, void *, void *);
-} quicklz_state;
-#endif
-
 /* Internal state for zlib */
 typedef struct zlib_state
 {
@@ -211,193 +188,6 @@ callCompressionValidator(PGFunction func, char *comptype, int32 complevel,
 	(void)DirectFunctionCall1(func, PointerGetDatum(&sa));
 }
 
-#ifdef HAVE_QUICKLZ
-/*
- * quicklz helper function.
- */
-static size_t
-quicklz_desired_sz(size_t input)
-{
-	/*
-	 * From the QuickLZ manual:
-	 *
-	 *   "The destination buffer must be at least size + 400 bytes large because
-	 *   incompressible data may increase in size."
-	 *
-	 */
-	return input + 400;
-}
-
-/*
- * Wrap up the qlz functions since C's type checking is getting in the
- * way.
- */
-static size_t
-quicklz_compressor(int level, const void *source, char *destination,
-				   size_t size, void *state)
-{
-	if (level == 1)
-	{
-		return qlz1_compress(source, destination, size,
-							 (qlz1_state_compress *)state);
-	}
-	else if (level == 3)
-	{
-		return qlz3_compress(source, destination, size,
-							 (qlz3_state_compress *)state);
-
-	}
-	Insist(false);
-}
-
-static size_t
-quicklz_decompressor(int level, const char *source, void *destination,
-					 void *state)
-{
-	if (level == 1)
-	{
-		return qlz1_decompress(source, destination,
-							   (qlz1_state_decompress *)state);
-	}
-	else if (level == 3)
-	{
-		return qlz3_decompress(source, destination,
-							   (qlz3_state_decompress *)state);
-	}
-	Insist(false);
-}
-
-/* ---------------------------------------------------------------------
- * Quicklz constructor and destructor
- * ---------------------------------------------------------------------
- */
-Datum
-quicklz_constructor(PG_FUNCTION_ARGS)
-{
-	TupleDesc td 			= PG_GETARG_POINTER(0);
-	StorageAttributes *sa	= PG_GETARG_POINTER(1);
-	CompressionState *cs 	= palloc0(sizeof(CompressionState));
-	quicklz_state *state	= palloc0(sizeof(quicklz_state));
-	bool compress			= PG_GETARG_BOOL(2);
-	size_t scratchlen		= 0;
-
-	cs->opaque = (void *)state;
-
-	Insist(PointerIsValid(td));
-	Insist(PointerIsValid(sa->comptype));
-	Insist(strcmp(sa->comptype, "quicklz") == 0);
-	Insist(sa->complevel == 1 ||
-		   sa->complevel == 3);
-
-	state->level = sa->complevel;
-	state->compress = compress;
-	if (sa->complevel == 1)
-	{
-		state->compress_fn = quicklz_compressor;
-		state->decompress_fn = quicklz_decompressor;
-		if (compress)
-			scratchlen = sizeof(qlz1_state_compress);
-		else
-			scratchlen = sizeof(qlz1_state_decompress);
-	}
-	else if (sa->complevel == 3)
-	{
-		state->compress_fn = quicklz_compressor;
-	   	state->decompress_fn = quicklz_decompressor;
-
-		if (compress)
-			scratchlen = sizeof(qlz3_state_compress);
-		else
-			scratchlen = sizeof(qlz3_state_decompress);
-	}
-	else
-		Insist(false); /* shouldn't get here but code defensively */
-
-	state->scratch = palloc0(scratchlen);
-
-	cs->desired_sz = quicklz_desired_sz;
-
-	PG_RETURN_POINTER(cs);
-}
-
-Datum
-quicklz_destructor(PG_FUNCTION_ARGS)
-{
-	CompressionState *cs = PG_GETARG_POINTER(0);
-	quicklz_state *state = (quicklz_state *) cs->opaque;
-
-	Insist(PointerIsValid(cs->opaque));
-
-	pfree(state->scratch);
-	pfree(cs->opaque);
-
-	PG_RETURN_VOID();
-}
-
-/* ---------------------------------------------------------------------
- * SQL invokable compression and decompression routines for built in
- * compression algorithms. All routines have the same SQL signature:
- *
- * void fun(internal, int, internal, int, internal, internal)
- *
- * If we were to think of this as a C function it would be more like:
- *
- * void fun(void *src, size_t src_sz, void *dst, size_t dst_sz,
- *		  size_t *dst_used, void *opaque)
- *
- * The meaning of each argument is as follows:
- * src - A pointer to data to be compressed/decompressed
- * src_sz - The number of bytes to compress/decompress
- * dst - A pointer to pre-allocated memory. The data compressed or
- * 		 decompressed by the function are written here.
- * dst_sz - The amount of memory in bytes allocated at dst
- * dst_used - The number of bytes written. If dst_sz was too small to
- *			store the data, this is set to zero.
- * opaque - Internal to the compression function.
- */
-Datum
-quicklz_compress(PG_FUNCTION_ARGS)
-{
-	const void *src 		  = PG_GETARG_POINTER(0);
-	int32 src_sz				= PG_GETARG_INT32(1);
-	void *dst					  = PG_GETARG_POINTER(2);
-	int32 dst_sz				= PG_GETARG_INT32(3);
-	int32 *dst_used			  = PG_GETARG_POINTER(4);
-	CompressionState *cs 	= (CompressionState *)PG_GETARG_POINTER(5);
-	quicklz_state *state	= (quicklz_state *)cs->opaque;
-
-	Insist(dst_sz >= quicklz_desired_sz(src_sz));
-
-	*dst_used = state->compress_fn(state->level, src, dst, (size_t)src_sz,
-								   state->scratch);
-
-	PG_RETURN_VOID();
-}
-
-Datum
-quicklz_decompress(PG_FUNCTION_ARGS)
-{
-	const char *src 	  	= PG_GETARG_POINTER(0);
-	int32 src_sz				= PG_GETARG_INT32(1);
-	void *dst					= PG_GETARG_POINTER(2);
-	int32 dst_sz			 	= PG_GETARG_INT32(3);
-	int32 *dst_used			  = PG_GETARG_POINTER(4);
-	CompressionState *cs 	= (CompressionState *)PG_GETARG_POINTER(5);
-	quicklz_state *state	= (quicklz_state *)cs->opaque;
-
-	Insist(src_sz > 0 && dst_sz > 0);
-	*dst_used = state->decompress_fn(state->level, src, dst, state->scratch);
-
-	PG_RETURN_VOID();
-}
-
-Datum
-quicklz_validator(PG_FUNCTION_ARGS)
-{
-	PG_RETURN_VOID();
-}
-#endif
-
 Datum
 zlib_constructor(PG_FUNCTION_ARGS)
 {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/backend/catalog/quicklz_compression.c
----------------------------------------------------------------------
diff --git a/src/backend/catalog/quicklz_compression.c b/src/backend/catalog/quicklz_compression.c
new file mode 100644
index 0000000..c6aabe5
--- /dev/null
+++ b/src/backend/catalog/quicklz_compression.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2011 EMC Corporation All Rights Reserved
+ *
+ * This software is protected, without limitation, by copyright law
+ * and international treaties. Use of this software and the intellectual
+ * property contained therein is expressly limited to the terms and
+ * conditions of the License Agreement under which it is provided by
+ * or on behalf of EMC.
+ *
+ * ---------------------------------------------------------------------
+ *
+ * The quicklz implementation is not provided due to licensing issues.  The
+ * following stub implementation is built if a proprietary implementation is
+ * not provided.
+ *
+ * Note that other compression algorithms actually work better for new
+ * installations anyway.
+ */
+
+#include "postgres.h"
+#include "utils/builtins.h"
+
+Datum
+quicklz_constructor(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "quicklz compression not supported");
+	PG_RETURN_VOID();
+}
+
+Datum
+quicklz_destructor(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "quicklz compression not supported");
+	PG_RETURN_VOID();
+}
+
+Datum
+quicklz_compress(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "quicklz compression not supported");
+	PG_RETURN_VOID();
+}
+
+Datum
+quicklz_decompress(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "quicklz compression not supported");
+	PG_RETURN_VOID();
+}
+
+Datum
+quicklz_validator(PG_FUNCTION_ARGS)
+{
+	elog(ERROR, "quicklz compression not supported");
+	PG_RETURN_VOID();
+}
+
+

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/include/catalog/pg_compression.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_compression.h b/src/include/catalog/pg_compression.h
index 6650eba..6c92be7 100644
--- a/src/include/catalog/pg_compression.h
+++ b/src/include/catalog/pg_compression.h
@@ -104,6 +104,8 @@ typedef FormData_pg_compression *Form_pg_compression;
 /* Initial contents */
 DATA(insert OID = 3060 ( zlib gp_zlib_constructor gp_zlib_destructor gp_zlib_compress gp_zlib_decompress
gp_zlib_validator PGUID ));
 
+DATA(insert OID = 3061 ( quicklz gp_quicklz_constructor gp_quicklz_destructor gp_quicklz_compress
gp_quicklz_decompress gp_quicklz_validator PGUID ));
+
 DATA(insert OID = 3062 ( rle_type gp_rle_type_constructor gp_rle_type_destructor gp_rle_type_compress
gp_rle_type_decompress gp_rle_type_validator PGUID ));
 
 DATA(insert OID = 3063 ( none gp_dummy_compression_constructor gp_dummy_compression_destructor
gp_dummy_compression_compress gp_dummy_compression_decompress gp_dummy_compression_validator
PGUID ));

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/include/catalog/pg_proc.h
----------------------------------------------------------------------
diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h
index eeb7bd6..64b7661 100644
--- a/src/include/catalog/pg_proc.h
+++ b/src/include/catalog/pg_proc.h
@@ -10275,6 +10275,26 @@ DESCR("anytable type serialization input function");
 DATA(insert OID = 3055 ( anytable_out  PGNSP PGUID 12 f f t f i 1 2275 f "3053" _null_ _null_
_null_ anytable_out - _null_ n ));
 DESCR("anytable type serialization output function");
 
+/* gp_quicklz_constructor(internal, internal, bool) => internal */
+DATA(insert OID = 5076 ( gp_quicklz_constructor  PGNSP PGUID 12 f f f f v 3 2281 f "2281
2281 16" _null_ _null_ _null_ quicklz_constructor - _null_ n ));
+DESCR("quicklz constructor");
+
+/* gp_quicklz_destructor(internal) => void */
+DATA(insert OID = 5077 ( gp_quicklz_destructor  PGNSP PGUID 12 f f f f v 1 2278 f "2281"
_null_ _null_ _null_ quicklz_destructor - _null_ n ));
+DESCR("quicklz destructor");
+
+/* gp_quicklz_compress(internal, int4, internal, int4, internal, internal) => void */
+DATA(insert OID = 5078 ( gp_quicklz_compress  PGNSP PGUID 12 f f f f i 6 2278 f "2281 23
2281 23 2281 2281" _null_ _null_ _null_ quicklz_compress - _null_ n ));
+DESCR("quicklz compressor");
+
+/* gp_quicklz_decompress(internal, int4, internal, int4, internal, internal) => void */
+DATA(insert OID = 5079 ( gp_quicklz_decompress  PGNSP PGUID 12 f f f f i 6 2278 f "2281 23
2281 23 2281 2281" _null_ _null_ _null_ quicklz_decompress - _null_ n ));
+DESCR("quicklz decompressor");
+
+/* gp_quicklz_validator(internal) => void */
+DATA(insert OID = 9925 ( gp_quicklz_validator  PGNSP PGUID 12 f f f f i 1 2278 f "2281" _null_
_null_ _null_ quicklz_validator - _null_ n ));
+DESCR("quicklz compression validator");
+
 /* gp_zlib_constructor(internal, internal, bool) => internal */ 
 DATA(insert OID = 9910 ( gp_zlib_constructor  PGNSP PGUID 12 f f f f v 3 2281 f "2281 2281
16" _null_ _null_ _null_ zlib_constructor - _null_ n ));
 DESCR("zlib constructor");

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/fdd2ae17/src/include/storage/gp_compress.h
----------------------------------------------------------------------
diff --git a/src/include/storage/gp_compress.h b/src/include/storage/gp_compress.h
index a3df970..08484cf 100644
--- a/src/include/storage/gp_compress.h
+++ b/src/include/storage/gp_compress.h
@@ -12,11 +12,6 @@
 
 #include "catalog/pg_compression.h"
 
-#ifdef HAVE_QUICKLZ
-#include <quicklz1.h>
-#include <quicklz3.h>
-#endif
-
 #ifdef HAVE_LIBZ
 #include <zlib.h>
 #endif


Mime
View raw message