hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From esp...@apache.org
Subject [10/50] [abbrv] incubator-hawq git commit: HAWQ-1256. Enhance libcurl connection to Ranger Plugin Service, keep it as a long-live connection in session level
Date Fri, 03 Feb 2017 09:00:12 GMT
HAWQ-1256. Enhance libcurl connection to Ranger Plugin Service, keep it as a long-live connection
in session level


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ad718734
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ad718734
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ad718734

Branch: refs/heads/2.1.0.0-incubating
Commit: ad718734898da2128ad47689243cc08043035573
Parents: 2f5910f
Author: stanlyxiang <stanly.sxiang@gmail.com>
Authored: Tue Jan 10 11:08:01 2017 +0800
Committer: Wen Lin <wlin@pivotal.io>
Committed: Wed Jan 11 15:51:32 2017 +0800

----------------------------------------------------------------------
 src/backend/libpq/rangerrest.c | 78 ++++++++++++++++---------------------
 src/backend/tcop/postgres.c    | 41 +++++++++++++++++++
 src/include/utils/rangerrest.h |  7 +++-
 3 files changed, 81 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ad718734/src/backend/libpq/rangerrest.c
----------------------------------------------------------------------
diff --git a/src/backend/libpq/rangerrest.c b/src/backend/libpq/rangerrest.c
index 56d30b5..fd8937a 100644
--- a/src/backend/libpq/rangerrest.c
+++ b/src/backend/libpq/rangerrest.c
@@ -71,9 +71,11 @@ static void getClientIP(char *remote_host)
 RangerACLResult parse_ranger_response(char* buffer)
 {
 	if (buffer == NULL || strlen(buffer) == 0)
+	{
 		return RANGERCHECK_UNKNOWN;
+	}
 
-	elog(LOG, "read from Ranger Restful API: %s", buffer);
+	elog(DEBUG3, "parse ranger restful response content : %s", buffer);
 
 	struct json_object *response = json_tokener_parse(buffer);
 	if (response == NULL) 
@@ -90,7 +92,7 @@ RangerACLResult parse_ranger_response(char* buffer)
 	}
 
 	int arraylen = json_object_array_length(accessObj);
-	elog(LOG, "Array Length: %d",arraylen);
+	elog(DEBUG3, "parse ranger response result array length: %d",arraylen);
 
 	// here should return which table's acl check failed in future.
 	for (int i=0; i< arraylen; i++){
@@ -161,7 +163,7 @@ json_object *create_ranger_request_json(List *args)
 		AclObjectKind kind = arg_ptr->kind;
 		char* object = arg_ptr->object;
 		Assert(user != NULL && object != NULL && privilege != NULL && arg_ptr->isAll);
-		elog(LOG, "build json for ranger request, user:%s, kind:%s, object:%s",
+		elog(DEBUG3, "build json for ranger restful request, user:%s, kind:%s, object:%s",
 				user, AclObjectKindStr[kind], object);
 
 		json_object *jelement = json_object_new_object();
@@ -281,29 +283,29 @@ static size_t write_callback(char *contents, size_t size, size_t nitems,
 	CURL_HANDLE curl = (CURL_HANDLE) userp;
 	Assert(curl != NULL);
 
-	if (curl->response.buffer == NULL) 
+	elog(DEBUG3, "ranger restful response size is %d. response buffer size is %d.", curl->response.response_size,
curl->response.buffer_size);
+	int original_size = curl->response.buffer_size;
+	while(curl->response.response_size + realsize >= curl->response.buffer_size)
 	{
-		curl->response.buffer = palloc0(realsize + 1);
+		/*double the buffer size if the buffer is not enough.*/
+		curl->response.buffer_size = curl->response.buffer_size * 2;
 	}
-	else 
+	if(original_size < curl->response.buffer_size)
 	{
-		/*Note:*/
-		/*our repalloc is not same as realloc, repalloc's first param(buffer) can not be NULL*/
-		curl->response.buffer = repalloc(curl->response.buffer, curl->response.size +
realsize + 1);
+		/* our repalloc is not same as realloc, repalloc's first param(buffer) can not be NULL
*/
+		curl->response.buffer = repalloc(curl->response.buffer, curl->response.buffer_size);
 	}
-
+	elog(DEBUG3, "ranger restful response size is %d. response buffer size is %d.", curl->response.response_size,
curl->response.buffer_size);
 	if (curl->response.buffer == NULL)
 	{
 		/* out of memory! */
 		elog(WARNING, "not enough memory for Ranger response");
 		return 0;
 	}
-
-	memcpy(curl->response.buffer + curl->response.size, contents, realsize);
-	curl->response.size += realsize;
-	curl->response.buffer[curl->response.size] = '\0';
-	elog(LOG, "read from Ranger Restful API: %s", curl->response.buffer);
-
+	memcpy(curl->response.buffer + curl->response.response_size, contents, realsize);
+	elog(DEBUG3, "read from ranger restful response: %s", curl->response.buffer);
+	curl->response.response_size += realsize;
+	curl->response.buffer[curl->response.response_size] = '\0';
 	return realsize;
 }
 
@@ -316,15 +318,14 @@ int call_ranger_rest(CURL_HANDLE curl_handle, const char* request)
 	CURLcode res;
 	Assert(request != NULL);
 
-	curl_global_init(CURL_GLOBAL_ALL);
-
-	/* init the curl session */
-	curl_handle->curl_handle = curl_easy_init();
-	if (curl_handle->curl_handle == NULL)
-	{
-		goto _exit;
-	}
-
+	/*
+	 * Re-initializes all options previously set on a specified CURL handle
+	 * to the default values. This puts back the handle to the same state as
+	 * it was in when it was just created with curl_easy_init.It does not
+	 * change the following information kept in the handle: live connections,
+	 * the Session ID cache, the DNS cache, the cookies and shares.
+	 */
+	curl_easy_reset(curl_handle->curl_handle);
 	/* timeout: hard-coded temporarily and maybe should be a guc in future */
 	curl_easy_setopt(curl_handle->curl_handle, CURLOPT_TIMEOUT, 30L);
 
@@ -364,19 +365,10 @@ int call_ranger_rest(CURL_HANDLE curl_handle, const char* request)
 	else
 	{
 		ret = 0;
-		elog(LOG, "%d bytes retrieved from Ranger Restful API.",
-			curl_handle->response.size);
-	}
-
-_exit:
-	/* cleanup curl stuff */
-	if (curl_handle->curl_handle)
-	{
-		curl_easy_cleanup(curl_handle->curl_handle);
+		elog(DEBUG3, "retrieved %d bytes from ranger restful response.",
+			curl_handle->response.response_size);
 	}
 
-	/* we're done with libcurl, so clean it up */
-	curl_global_cleanup();
 	return ret;
 }
 
@@ -388,13 +380,11 @@ int check_privilege_from_ranger(List *arg_list)
 	json_object* jrequest = create_ranger_request_json(arg_list);
 	Assert(jrequest != NULL);
 	const char *request = json_object_to_json_string(jrequest);
-	elog(LOG, "Send JSON request to Ranger: %s", request);
+	elog(DEBUG3, "send json request to ranger : %s", request);
 	Assert(request != NULL);
-	struct curl_context_t curl_context;
-	memset(&curl_context, 0, sizeof(struct curl_context_t));
 
 	/* call GET method to send request*/
-	if (call_ranger_rest(&curl_context, request) < 0)
+	if (call_ranger_rest(&curl_context_ranger, request) < 0)
 	{
 		return RANGERCHECK_NO_PRIV;
 	}
@@ -403,11 +393,11 @@ int check_privilege_from_ranger(List *arg_list)
 	json_object_put(jrequest);
 
 	/* parse the JSON-format result */
-	RangerACLResult ret = parse_ranger_response(curl_context.response.buffer);
-	/* free response buffer */
-	if (curl_context.response.buffer != NULL)
+	RangerACLResult ret = parse_ranger_response(curl_context_ranger.response.buffer);
+	if (curl_context_ranger.response.buffer != NULL)
 	{
-		pfree(curl_context.response.buffer);
+		/* reset response size to reuse the buffer. */
+		curl_context_ranger.response.response_size = 0;
 	}
 
 	return ret;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ad718734/src/backend/tcop/postgres.c
----------------------------------------------------------------------
diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c
index 5a8327e..c8d7e33 100644
--- a/src/backend/tcop/postgres.c
+++ b/src/backend/tcop/postgres.c
@@ -114,6 +114,8 @@
 
 #include "cdb/cdbinmemheapam.h"
 
+#include "utils/rangerrest.h"
+
 #include "resourcemanager/dynrm.h"
 #include "resourcemanager/envswitch.h"
 #include "resourcemanager/communication/rmcomm_QD2RM.h"
@@ -133,6 +135,7 @@ extern char *optarg;
 extern char *savedSeqServerHost;
 extern int savedSeqServerPort;
 
+struct curl_context_t curl_context_ranger;
 /* ----------------
  *		global variables
  * ----------------
@@ -266,6 +269,7 @@ static void log_disconnections(int code, Datum arg);
 static bool renice_current_process(int nice_level);
 static int getSlaveHostNumber(FILE *fp);
 static bool CheckSlaveFile();
+static void curl_finalize(int code, Datum arg);
 
 /*saved interrupt global variable for client_read_xxx functions*/
 static bool SavedImmediateInterruptOK = false;
@@ -4626,6 +4630,25 @@ PostgresMain(int argc, char *argv[], const char *username)
 	if (!ignore_till_sync)
 		send_ready_for_query = true;	/* initially, or after error */
 
+	/* for enable ranger*/
+	if (AmIMaster() && enable_ranger && !curl_context_ranger.hasInited)
+	{
+		memset(&curl_context_ranger, 0, sizeof(curl_context_t));
+		curl_global_init(CURL_GLOBAL_ALL);
+		/* init the curl session */
+		curl_context_ranger.curl_handle = curl_easy_init();
+		if (curl_context_ranger.curl_handle == NULL) {
+			/* cleanup curl stuff */
+			/* no need to cleanup curl_handle since it's null. just cleanup curl global.*/
+			curl_global_cleanup();
+			elog(ERROR, "initialize global curl context failed.");
+		}
+		curl_context_ranger.hasInited = true;
+		curl_context_ranger.response.buffer = palloc0(CURL_RES_BUFFER_SIZE);
+		curl_context_ranger.response.buffer_size = CURL_RES_BUFFER_SIZE;
+		elog(DEBUG3, "initialize global curl context for privileges check.");
+		on_proc_exit(curl_finalize, 0);
+	}
 	/*
 	 * Non-error queries loop here.
 	 */
@@ -5314,6 +5337,24 @@ PostgresMain(int argc, char *argv[], const char *username)
 	return 1;					/* keep compiler quiet */
 }
 
+static void
+curl_finalize(int code, Datum arg __MAYBE_UNUSED)
+{
+	if (AmIMaster() && curl_context_ranger.hasInited)
+	{
+		if (curl_context_ranger.response.buffer != NULL) {
+			pfree(curl_context_ranger.response.buffer);
+		}
+		/* cleanup curl stuff */
+		if (curl_context_ranger.curl_handle) {
+			curl_easy_cleanup(curl_context_ranger.curl_handle);
+		}
+		/* we're done with libcurl, so clean it up */
+		curl_global_cleanup();
+		curl_context_ranger.hasInited = false;
+		elog(DEBUG3, "finalize the global struct for curl handle context.");
+	}
+}
 
 /*
  * Obtain platform stack depth limit (in bytes)

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ad718734/src/include/utils/rangerrest.h
----------------------------------------------------------------------
diff --git a/src/include/utils/rangerrest.h b/src/include/utils/rangerrest.h
index 692c832..f67d8e5 100644
--- a/src/include/utils/rangerrest.h
+++ b/src/include/utils/rangerrest.h
@@ -37,6 +37,7 @@
 #include "tcop/tcopprot.h"
 
 #define HOST_BUFFER_SIZE 1025
+#define CURL_RES_BUFFER_SIZE 1024
 
 typedef enum
 {
@@ -59,10 +60,13 @@ typedef struct curl_context_t
   struct
   {
     char* buffer;
-    int size;
+    int response_size;
+    int buffer_size;
   } response;
 
   char* last_http_reponse;
+
+  bool hasInited;
 } curl_context_t;
 
 typedef curl_context_t* CURL_HANDLE;
@@ -94,5 +98,6 @@ RangerACLResult parse_ranger_response(char *);
 json_object *create_ranger_request_json(List *);
 int call_ranger_rest(CURL_HANDLE curl_handle, const char *request);
 extern int check_privilege_from_ranger(List *);
+extern struct curl_context_t curl_context_ranger;
 
 #endif


Mime
View raw message