hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject incubator-hawq git commit: HAWQ-1099. Output yaml file should not contain Bucketnum attribute with random distributed table.
Date Tue, 18 Oct 2016 05:38:27 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 676eb828c -> c8ef3a0ac


HAWQ-1099. Output yaml file should not contain Bucketnum attribute with random distributed
table.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/c8ef3a0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/c8ef3a0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/c8ef3a0a

Branch: refs/heads/master
Commit: c8ef3a0ac847743dbed0d0715b6534e34f95d92a
Parents: 676eb82
Author: xunzhang <xunzhangthu@gmail.com>
Authored: Thu Oct 13 18:25:40 2016 +0800
Committer: Wen Lin <wlin@pivotal.io>
Committed: Tue Oct 18 13:37:51 2016 +0800

----------------------------------------------------------------------
 tools/bin/hawqextract  |  6 ++++--
 tools/bin/hawqregister | 15 ++++++++++-----
 2 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c8ef3a0a/tools/bin/hawqextract
----------------------------------------------------------------------
diff --git a/tools/bin/hawqextract b/tools/bin/hawqextract
index 28f55d6..3a6ef66 100644
--- a/tools/bin/hawqextract
+++ b/tools/bin/hawqextract
@@ -421,7 +421,8 @@ def extract_metadata(conn, tbname):
         logger.info('-- extract Distribution_Policy')
         metadata['Distribution_Policy'] = accessor.get_distribution_policy_info(rel_pgclass['oid'],
relid)
         logger.info('-- extract bucket number')
-        metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid'])
+        if accessor.get_distribution_policy_info(rel_pgclass['oid'], relid).startswith('DISTRIBUTED
BY'):
+            metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid'])
 
     def extract_Parquet_metadata():
         relid = rel_pgclass['oid']
@@ -471,7 +472,8 @@ def extract_metadata(conn, tbname):
         logger.info('-- extract Distribution_Policy')
         metadata['Distribution_Policy'] = accessor.get_distribution_policy_info(rel_pgclass['oid'],
relid)
         logger.info('-- extract bucket number')
-        metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid'])
+        if accessor.get_distribution_policy_info(rel_pgclass['oid'], relid).startswith('DISTRIBUTED
BY'):
+            metadata['Bucketnum'] = accessor.get_bucket_number(rel_pgclass['oid'])
 
 
     # extract AO/Parquet specific metadata

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/c8ef3a0a/tools/bin/hawqregister
----------------------------------------------------------------------
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index 29b3a30..a1c8e29 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -60,14 +60,18 @@ def option_parser():
 
 def register_yaml_dict_check(D, table_column_num, src_tablename):
     '''check exists'''
-    check_list = ['DFS_URL', 'Distribution_Policy', 'FileFormat', 'TableName', 'Bucketnum']
+    check_list = ['DFS_URL', 'Distribution_Policy', 'FileFormat', 'TableName']
     for attr in check_list:
         if D.get(attr) is None:
             logger.error('Wrong configuration yaml file format: "%s" attribute does not exist.\n
See example in "hawq register --help".' % attr)
             sys.exit(1)
-    if D['Bucketnum'] <= 0:
-        logger.error('Bucketnum should not be zero, please check your yaml configuration
file.')
-        sys.exit(1)
+    if D['Distribution_Policy'].startswith('DISTRIBUTED BY'):
+        if D.get('Bucketnum') is None:
+            logger.error('Wrong configuration yaml file format: "%s" attribute does not exist.\n
See example in "hawq register --help".' % attr)
+            sys.exit(1)
+        if D['Bucketnum'] <= 0:
+            logger.error('Bucketnum should not be zero, please check your yaml configuration
file.')
+            sys.exit(1)
     if D['FileFormat'] in ['Parquet', 'AO']:
         prefix = D['FileFormat']
         local_check_list = ['%s_FileLocations' % prefix, '%s_Schema' % prefix]
@@ -432,7 +436,8 @@ class HawqRegister(object):
         if len(params[Format_FileLocations]['Files']):
             files, sizes = [params['DFS_URL'] + d['path'] for d in params[Format_FileLocations]['Files']],
[d['size'] for d in params[Format_FileLocations]['Files']]
         encoding = params['Encoding']
-        self._set_yml_dataa(Format, files, sizes, params['TableName'], params['%s_Schema'
% Format], params['Distribution_Policy'], params[Format_FileLocations], params['Bucketnum'],
partitionby,\
+        bucketNum = params['Bucketnum'] if params['Distribution_Policy'].startswith('DISTRIBUTED
BY') else 6
+        self._set_yml_dataa(Format, files, sizes, params['TableName'], params['%s_Schema'
% Format], params['Distribution_Policy'], params[Format_FileLocations], bucketNum, partitionby,\
                       partitions_constraint, partitions_name, partitions_compression_level,
partitions_compression_type, partitions_checksum, partitions_filepaths, partitions_filesizes,
encoding)
 
 


Mime
View raw message