hawq-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From huor <...@git.apache.org>
Subject [GitHub] incubator-hawq pull request #1384: HAWQ-1628. Add HDFS protocol for external...
Date Wed, 08 Aug 2018 01:56:02 GMT
Github user huor commented on a diff in the pull request:

    https://github.com/apache/incubator-hawq/pull/1384#discussion_r208435664
  
    --- Diff: src/backend/commands/tablecmds.c ---
    @@ -939,38 +983,217 @@ DefineExternalRelation(CreateExternalStmt *createExtStmt)
     	char*					  commandString = NULL;
     	char					  rejectlimittype = '\0';
     	char					  formattype;
    +	char*					  formattername = NULL;
     	int						  rejectlimit = -1;
     	int						  encoding = -1;
     	int preferred_segment_num = -1;
     	bool					  issreh = false; /* is single row error handling requested? */
    +	bool					  isexternal = createExtStmt->isexternal;
     	bool					  iswritable = createExtStmt->iswritable;
     	bool					  isweb = createExtStmt->isweb;
    +	bool 					  forceCreateDir = createExtStmt->forceCreateDir;
    +
    +	bool 					  isExternalHdfs = false;
    +	bool 					  isExternalMagma = false;
    +	bool						  isExternalHive = false;
    +	char*					  location = NULL;
    +	int						  location_len = NULL;
     
     	/*
     	 * now set the parameters for keys/inheritance etc. Most of these are
     	 * uninteresting for external relations...
     	 */
    -	createStmt->relation = createExtStmt->relation;
    -	createStmt->tableElts = createExtStmt->tableElts;
    -	createStmt->inhRelations = NIL;
    -	createStmt->constraints = NIL;
    -	createStmt->options = NIL;
    -	createStmt->oncommit = ONCOMMIT_NOOP;
    -	createStmt->tablespacename = NULL;
    +	createStmt->base = createExtStmt->base;
    +	// external table options is not compatible with internal table
    +	// set NIL here
    +	createStmt->base.options = NIL;
     	createStmt->policy = createExtStmt->policy; /* policy was set in transform */
    -	
    +
    +	 /*
    +	 * Recognize formatter option if there are some tokens found in parser.
    +	 * This design is to give CREATE EXTERNAL TABLE DDL the flexiblity to
    +	 * support user defined external formatter options.
    +	 */
    +	recognizeExternalRelationFormatterOptions(createExtStmt);
    +
    +	/*
    +	 * Get tablespace, database, schema for the relation
    +	 */
    +	RangeVar *rel = createExtStmt->base.relation;
    +	// get tablespace name for the relation
    +	Oid tablespace_id = (gp_upgrade_mode) ? DEFAULTTABLESPACE_OID : GetDefaultTablespace();
    +	if (!OidIsValid(tablespace_id))
    +	{
    +		tablespace_id = get_database_dts(MyDatabaseId);
    +	}
    +	char *tablespace_name = get_tablespace_name(tablespace_id);
    +
    +	// get database name for the relation
    +	char *database_name = rel->catalogname ? rel->catalogname : get_database_name(MyDatabaseId);
    +
    +	// get schema name for the relation
    +	char *schema_name = get_namespace_name(RangeVarGetCreationNamespace(rel));
    +
    +	// get table name for the relation
    +	char *table_name = rel->relname;
    +
    +	/*
    +	 * Do some special logic when we use custom
    +	 */
    +	if (exttypeDesc->exttabletype == EXTTBL_TYPE_LOCATION)
    +	{
    +		if (exttypeDesc->location_list == NIL)
    +		{
    +			if (dfs_url == NULL)
    +			{
    +				ereport(ERROR,
    +				        (errcode(ERRCODE_SYNTAX_ERROR),
    +				         errmsg("Cannot create table on HDFS when the service is not available"),
    +				         errhint("Check HDFS service and hawq_dfs_url configuration"),
    +				         errOmitLocation(true)));
    +			}
    +
    +			location_len = strlen(PROTOCOL_HDFS) +         /* hdfs:// */
    +			               strlen(dfs_url) +               /* hawq_dfs_url */
    +			               // 1 + strlen(filespace_name) + /* '/' + filespace name */
    +			               1 + strlen(tablespace_name) +   /* '/' + tablespace name */
    +			               1 + strlen(database_name) +     /* '/' + database name */
    +			               1 + strlen(schema_name) +       /* '/' + schema name */
    +			               1 + strlen(table_name) + 1;     /* '/' + table name + '\0' */
    +
    +			char *path;
    +
    +			if (createExtStmt->parentPath == NULL) {
    +				path = (char *)palloc(sizeof(char) * location_len);
    +				sprintf(path, "%s%s/%s/%s/%s/%s",
    +						PROTOCOL_HDFS,
    +						dfs_url, /* filespace_name, */ tablespace_name,
    +						database_name, schema_name, table_name);
    +			}
    +			else {
    +				path = (char *)palloc(sizeof(char) *
    +						(location_len + strlen(createExtStmt->parentPath)+1));
    +				sprintf(path, "%s%s/%s/%s/%s/%s/%s",
    +						PROTOCOL_HDFS,
    +						dfs_url, /* filespace_name, */ tablespace_name,
    +						database_name, schema_name,
    +						createExtStmt->parentPath, table_name);
    +			}
    +
    +			exttypeDesc->location_list = list_make1((Node *) makeString(path));
    +		}
    +
    +		/* Check the location to extract protocol */
    +		ListCell   *first_uri = list_head(exttypeDesc->location_list);
    +		Value		*v = lfirst(first_uri);
    +		char		*uri_str = pstrdup(v->val.str);
    +		Uri			*uri = ParseExternalTableUri(uri_str);
    +		bool 		isHdfs = is_hdfs_protocol(uri);
    +		bool 		isHive = is_hive_protocol(uri);
    +
    +		pfree(uri_str);
    +		FreeExternalTableUri(uri);
    +
    +		if (isHdfs)
    +		{
    +			/* We have an HDFS external protocol */
    +			isExternalHdfs = true;
    +		}
    +
    +		if (isHive)
    +		{
    +			/* We have an HIVE external protocol */
    +			isExternalHive = true;
    +		}
    +	}
    +
    +	if (isExternalHdfs)
    +	{
    +		if (list_length(exttypeDesc->location_list)!= 1)
    +			ereport(ERROR,
    +					(errcode(ERRCODE_SYNTAX_ERROR),
    +							errmsg("Only support 1 external HDFS location. "
    +									"Now input %d location(s)",
    +									list_length(exttypeDesc->location_list))));
    +	}
    +
    +	if (isExternalHive)
    +	{
    +		if (list_length(exttypeDesc->location_list)!= 1)
    +			ereport(ERROR,
    +					(errcode(ERRCODE_SYNTAX_ERROR),
    +							errmsg("Only support 1 external HIVE location. "
    +									"Now input %d location(s)",
    +									list_length(exttypeDesc->location_list))));
    +	}
    +
    +	if (isExternalMagma)
    --- End diff --
    
    No magma


---

Mime
View raw message