hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject incubator-hawq git commit: HAWQ-1127. HAWQ should print error message instead of python function stack when yaml file is invalid.
Date Mon, 31 Oct 2016 07:36:12 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 778ab999a -> 7e11e66a6


HAWQ-1127. HAWQ should print error message instead of python function stack when yaml file
is invalid.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/7e11e66a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/7e11e66a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/7e11e66a

Branch: refs/heads/master
Commit: 7e11e66a698dd4b87846c5680dab44df7b740ac6
Parents: 778ab99
Author: Wen Lin <wlin@pivotal.io>
Authored: Mon Oct 31 15:33:26 2016 +0800
Committer: Wen Lin <wlin@pivotal.io>
Committed: Mon Oct 31 15:33:26 2016 +0800

----------------------------------------------------------------------
 .../missing_constraint_partition.yml            | 295 +++++++++++++++++++
 .../test_hawq_register_usage2_case1.cpp         |   1 +
 tools/bin/hawqregister                          |  69 +++--
 3 files changed, 334 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e11e66a/src/test/feature/ManagementTool/missing_constraint_partition.yml
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/missing_constraint_partition.yml b/src/test/feature/ManagementTool/missing_constraint_partition.yml
new file mode 100755
index 0000000..89c537d
--- /dev/null
+++ b/src/test/feature/ManagementTool/missing_constraint_partition.yml
@@ -0,0 +1,295 @@
+AO_FileLocations:
+  Blocksize: 32768
+  Checksum: false
+  CompressionLevel: 0
+  CompressionType: null
+  Files:
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/1
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/2
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/3
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/4
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/5
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD@/6
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/1
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/2
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/3
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/4
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/5
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  - eofuncompressed: 0
+    path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW@/6
+    size: 0
+    tupcount: 0
+    varblockcount: 0
+  PartitionBy: PARTITION BY list (gender)
+  Partitions:
+  - Blocksize: 32768
+    Checksum: false
+    CompressionLevel: 0
+    CompressionType: null
+    Constraint: PARTITION girls VALUES('F') WITH (appendonly=true)
+    Files:
+    - eofuncompressed: 432
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/1
+      size: 432
+      tupcount: 16
+      varblockcount: 1
+    - eofuncompressed: 408
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/2
+      size: 408
+      tupcount: 15
+      varblockcount: 1
+    - eofuncompressed: 464
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/3
+      size: 464
+      tupcount: 17
+      varblockcount: 1
+    - eofuncompressed: 408
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/4
+      size: 408
+      tupcount: 15
+      varblockcount: 1
+    - eofuncompressed: 464
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/5
+      size: 464
+      tupcount: 17
+      varblockcount: 1
+    - eofuncompressed: 536
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD1@/6
+      size: 536
+      tupcount: 20
+      varblockcount: 1
+    - eofuncompressed: 280
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/1
+      size: 280
+      tupcount: 10
+      varblockcount: 1
+    - eofuncompressed: 224
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/2
+      size: 224
+      tupcount: 8
+      varblockcount: 1
+    - eofuncompressed: 224
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/3
+      size: 224
+      tupcount: 8
+      varblockcount: 1
+    - eofuncompressed: 256
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/4
+      size: 256
+      tupcount: 9
+      varblockcount: 1
+    - eofuncompressed: 200
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/5
+      size: 200
+      tupcount: 7
+      varblockcount: 1
+    - eofuncompressed: 224
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW1@/6
+      size: 224
+      tupcount: 8
+      varblockcount: 1
+    Name: nt_1_prt_girls
+  - Blocksize: 32768
+    Checksum: false
+    CompressionLevel: 0
+    CompressionType: null
+    Constraint: PARTITION boys VALUES('M') WITH (appendonly=true)
+    Files:
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/1
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/2
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/3
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/4
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/5
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD2@/6
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/1
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/2
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/3
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/4
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/5
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW2@/6
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    Name: t_1_prt_boys
+  - Blocksize: 32768
+    Checksum: false
+    CompressionLevel: 0
+    CompressionType: null
+    Name: nt_1_prt_boys
+  - Blocksize: 32768
+    Checksum: false
+    CompressionLevel: 0
+    CompressionType: null
+    Constraint: DEFAULT PARTITION other  WITH (appendonly=true)
+    Files:
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/1
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/2
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/3
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/4
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/5
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_OLD3@/6
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/1
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/2
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/3
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/4
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/5
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    - eofuncompressed: 0
+      path: /hawq_default/16385/@DATABASE_OID@/@TABLE_OID_NEW3@/6
+      size: 0
+      tupcount: 0
+      varblockcount: 0
+    Name: nt_1_prt_other
+AO_Schema:
+- name: id
+  type: int4
+- name: rank
+  type: int4
+- name: year
+  type: int4
+- name: gender
+  type: bpchar
+- name: count
+  type: int4
+Bucketnum: 6
+DBVersion: PostgreSQL 8.2.15 (Greenplum Database 4.2.0 build 1) (HAWQ 2.0.1.0 build
+  dev) on x86_64-apple-darwin14.5.0, compiled by GCC Apple LLVM version 6.1.0 (clang-602.0.53)
+  (based on LLVM 3.6.0svn) compiled on Jul  6 2016 10:22:33
+DFS_URL: hdfs://localhost:8020
+Distribution_Policy: DISTRIBUTED BY (id)
+Encoding: UTF8
+FileFormat: AO
+TableName: public.nt
+Version: 1.0.0

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e11e66a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
----------------------------------------------------------------------
diff --git a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
index 8411f0c..90aa8f0 100644
--- a/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
+++ b/src/test/feature/ManagementTool/test_hawq_register_usage2_case1.cpp
@@ -40,6 +40,7 @@ TEST_F(TestHawqRegister, TestUsage2Case1IncorrectYaml) {
     EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_checksum.yml xx"));
     EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "wrong_dfs_url.yml xx"));
     EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_bucketnum.yml xx"));
+    EXPECT_EQ(1, Command::getCommandStatus("hawq register -d " + (string) HAWQ_DB + " -c
" + filePath + "missing_constraint_partition.yml xx"));
 }
 
 TEST_F(TestHawqRegister, TestUsage2Case1MismatchFileNumber) {

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/7e11e66a/tools/bin/hawqregister
----------------------------------------------------------------------
diff --git a/tools/bin/hawqregister b/tools/bin/hawqregister
index b9b759e..356aef1 100755
--- a/tools/bin/hawqregister
+++ b/tools/bin/hawqregister
@@ -443,6 +443,7 @@ class HawqRegister(object):
             print e
             self.failure_handler.rollback()
             sys.exit(1)
+            
         table_column_num = self.accessor.get_table_column_num(self.tablename)
         register_yaml_dict_check(params, table_column_num, self.tablename)
         partitions_filepaths = []
@@ -457,37 +458,43 @@ class HawqRegister(object):
         partitions_varblockcounts = []
         files, sizes, tupcounts, eofuncompresseds, varblockcounts = [], [], [], [], []
 
-        if params['FileFormat'].lower() == 'parquet':
-            Format = 'Parquet'
-        else: #AO format
-            Format = 'AO'
-        Format_FileLocations = '%s_FileLocations' % Format
-        partitionby = params.get(Format_FileLocations).get('PartitionBy')
-        if params.get(Format_FileLocations).get('Partitions') and len(params[Format_FileLocations]['Partitions']):
-            partitions_checksum = [d['Checksum'] for d in params[Format_FileLocations]['Partitions']]
-            partitions_compression_level = [d['CompressionLevel'] for d in params[Format_FileLocations]['Partitions']]
-            partitions_compression_type = [d['CompressionType'] for d in params[Format_FileLocations]['Partitions']]
-            partitions_constraint = [d['Constraint'] for d in params[Format_FileLocations]['Partitions']]
-            partitions_files = [d['Files'] for d in params[Format_FileLocations]['Partitions']]
-            if len(partitions_files):
-                for pfile in partitions_files:
-                    partitions_filepaths.append([params['DFS_URL'] + item['path'] for item
in pfile])
-                    partitions_filesizes.append([item['size'] for item in pfile])
-                    partitions_tupcounts.append([item['tupcount'] if item.has_key('tupcount')
else -1 for item in pfile])
-                    partitions_eofuncompresseds.append([item['eofuncompressed'] if item.has_key('eofuncompressed')
else -1 for item in pfile])
-                    partitions_varblockcounts.append([item['varblockcount'] if item.has_key('varblockcount')
else -1 for item in pfile])
-            partitions_name = [d['Name'] for d in params[Format_FileLocations]['Partitions']]
-        if len(params[Format_FileLocations]['Files']):
-            for ele in params[Format_FileLocations]['Files']:
-                files.append(params['DFS_URL'] + ele['path'])
-                sizes.append(ele['size'])
-                tupcounts.append(ele['tupcount'] if ele.has_key('tupcount') else -1)
-                eofuncompresseds.append(ele['eofuncompressed'] if ele.has_key('eofuncompressed')
else -1)
-                varblockcounts.append(ele['varblockcount'] if ele.has_key('varblockcount')
else -1)
-
-        encoding = params['Encoding']
-        bucketNum = params['Bucketnum'] if params['Distribution_Policy'].startswith('DISTRIBUTED
BY') else 6
-        self._set_yml_data(Format, files, sizes, tupcounts, eofuncompresseds, varblockcounts,
params['TableName'], params['%s_Schema' % Format], params['Distribution_Policy'], params[Format_FileLocations],
bucketNum, partitionby, partitions_constraint, partitions_name, partitions_compression_level,
partitions_compression_type, partitions_checksum, partitions_filepaths, partitions_filesizes,
partitions_tupcounts, partitions_eofuncompresseds, partitions_varblockcounts, encoding)
+        try:
+            if params['FileFormat'].lower() == 'parquet':
+                Format = 'Parquet'
+            else: #AO format
+                Format = 'AO'
+            Format_FileLocations = '%s_FileLocations' % Format
+            partitionby = params.get(Format_FileLocations).get('PartitionBy')
+            if params.get(Format_FileLocations).get('Partitions') and len(params[Format_FileLocations]['Partitions']):
+                partitions_checksum = [d['Checksum'] for d in params[Format_FileLocations]['Partitions']]
+                partitions_compression_level = [d['CompressionLevel'] for d in params[Format_FileLocations]['Partitions']]
+                partitions_compression_type = [d['CompressionType'] for d in params[Format_FileLocations]['Partitions']]
+                partitions_constraint = [d['Constraint'] for d in params[Format_FileLocations]['Partitions']]
+                partitions_files = [d['Files'] for d in params[Format_FileLocations]['Partitions']]
+                if len(partitions_files):
+                    for pfile in partitions_files:
+                        partitions_filepaths.append([params['DFS_URL'] + item['path'] for
item in pfile])
+                        partitions_filesizes.append([item['size'] for item in pfile])
+                        partitions_tupcounts.append([item['tupcount'] if item.has_key('tupcount')
else -1 for item in pfile])
+                        partitions_eofuncompresseds.append([item['eofuncompressed'] if item.has_key('eofuncompressed')
else -1 for item in pfile])
+                        partitions_varblockcounts.append([item['varblockcount'] if item.has_key('varblockcount')
else -1 for item in pfile])
+                partitions_name = [d['Name'] for d in params[Format_FileLocations]['Partitions']]
+            if len(params[Format_FileLocations]['Files']):
+                for ele in params[Format_FileLocations]['Files']:
+                    files.append(params['DFS_URL'] + ele['path'])
+                    sizes.append(ele['size'])
+                    tupcounts.append(ele['tupcount'] if ele.has_key('tupcount') else -1)
+                    eofuncompresseds.append(ele['eofuncompressed'] if ele.has_key('eofuncompressed')
else -1)
+                    varblockcounts.append(ele['varblockcount'] if ele.has_key('varblockcount')
else -1)
+
+            encoding = params['Encoding']
+            bucketNum = params['Bucketnum'] if params['Distribution_Policy'].startswith('DISTRIBUTED
BY') else 6
+            self._set_yml_data(Format, files, sizes, tupcounts, eofuncompresseds, varblockcounts,
params['TableName'], params['%s_Schema' % Format], params['Distribution_Policy'], params[Format_FileLocations],
bucketNum, partitionby, partitions_constraint, partitions_name, partitions_compression_level,
partitions_compression_type, partitions_checksum, partitions_filepaths, partitions_filesizes,
partitions_tupcounts, partitions_eofuncompresseds, partitions_varblockcounts, encoding)
+
+        except KeyError as e:
+            print 'Invalid yaml file, %s is missing.' % e
+            self.failure_handler.rollback()
+            sys.exit(1)
 
 
     # check conflicting distributed policy


Mime
View raw message