hawq-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lil...@apache.org
Subject incubator-hawq git commit: HAWQ-1091. Fix HAWQ InputFormat Bugs.
Date Mon, 10 Oct 2016 05:57:31 GMT
Repository: incubator-hawq
Updated Branches:
  refs/heads/master 19f3fa81b -> 28d192d23


HAWQ-1091. Fix HAWQ InputFormat Bugs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/28d192d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/28d192d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/28d192d2

Branch: refs/heads/master
Commit: 28d192d2329dc68373616fe65530c590d501a7b1
Parents: 19f3fa8
Author: xunzhang <xunzhangthu@gmail.com>
Authored: Sun Oct 9 20:35:31 2016 +0800
Committer: Lili Ma <ictmalili@gmail.com>
Committed: Mon Oct 10 13:57:24 2016 +0800

----------------------------------------------------------------------
 .../com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java  | 2 ++
 .../pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java  | 6 ++++++
 .../test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java  | 2 +-
 .../hawq/mapreduce/pt/HAWQInputFormatPerformanceTest_TPCH.java | 2 +-
 .../hawq/mapreduce/ut/HAWQInputFormatUnitTest_TPCH.java        | 5 +++++
 5 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/28d192d2/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
----------------------------------------------------------------------
diff --git a/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
b/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
index 250eb86..5a46489 100644
--- a/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
+++ b/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataAccessor.java
@@ -111,6 +111,8 @@ public abstract class MetadataAccessor {
 		try {
 			return new MetadataYAMLAccessor(file);
 
+		} catch (IllegalStateException e) {
+		  throw new IllegalStateException(e.getMessage());
 		} catch (Exception e) {
 			throw new MetadataAccessException("error occurred when reading " + file, e);
 		}

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/28d192d2/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
----------------------------------------------------------------------
diff --git a/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
b/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
index 3e193b5..4570bc2 100644
--- a/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
+++ b/contrib/hawq-hadoop/hawq-mapreduce-common/src/main/java/com/pivotal/hawq/mapreduce/metadata/MetadataYAMLAccessor.java
@@ -158,6 +158,9 @@ class MetadataYAMLAccessor extends MetadataAccessor {
 			Map<?, ?> file = (Map<?, ?>) files.get(i);
 			String filePath = file.get("path").toString();
 			long fileLen = Long.parseLong(file.get("size").toString());
+			if (fileLen == 0) {
+			  throw new IllegalStateException("Empty file can not be processed by HAWQAOInputFormat
since it is an empty file.");
+			}
 
 			fileStatuses.add(new HAWQAOFileStatus(filePath, fileLen, checksum, compressType, blockSize));
 		}
@@ -198,6 +201,9 @@ class MetadataYAMLAccessor extends MetadataAccessor {
 			Map<?, ?> file = (Map<?, ?>) files.get(i);
 			final String filePath = file.get("path").toString();
 			final long fileLen = Long.parseLong(file.get("size").toString());
+			if (fileLen == 0) {
+			  throw new IllegalStateException("Empty file can not be processed by HAWQParquetInputFormat
since it is an empty file.");
+			}
 			fileStatuses.add(new HAWQFileStatus(filePath, fileLen));
 		}
 		return fileStatuses;

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/28d192d2/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java
----------------------------------------------------------------------
diff --git a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java
index 7368b61..19f34f1 100644
--- a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java
+++ b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/TPCHLocalTester.java
@@ -66,7 +66,7 @@ public class TPCHLocalTester extends TPCHTester {
 
 				// load TPCH data
 				Map<String, String> rs = HAWQJdbcUtils.executeSafeQueryForSingleRow(
-						conn, "SELECT COUNT(*) segnum FROM gp_segment_configuration WHERE content>=0;");
+						conn, "SELECT COUNT(*) segnum FROM gp_segment_configuration WHERE role='p';");
 				int segnum = Integer.parseInt(rs.get("segnum"));
 				MRFormatTestUtils.runShellCommand(tpchSpec.getLoadCmd(segnum));
 

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/28d192d2/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/pt/HAWQInputFormatPerformanceTest_TPCH.java
----------------------------------------------------------------------
diff --git a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/pt/HAWQInputFormatPerformanceTest_TPCH.java
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/pt/HAWQInputFormatPerformanceTest_TPCH.java
index 73bf109..aa51330 100644
--- a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/pt/HAWQInputFormatPerformanceTest_TPCH.java
+++ b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/pt/HAWQInputFormatPerformanceTest_TPCH.java
@@ -151,7 +151,7 @@ public class HAWQInputFormatPerformanceTest_TPCH extends Configured implements
T
 		try {
 			conn = MRFormatTestUtils.getTestDBConnection();
 			Map<String, String> rs = HAWQJdbcUtils.executeSafeQueryForSingleRow(
-					conn, "SELECT COUNT(*) segnum FROM gp_segment_configuration WHERE content>=0;");
+					conn, "SELECT COUNT(*) segnum FROM gp_segment_configuration WHERE role='p';");
 			segnum = Integer.parseInt(rs.get("segnum"));
 		} finally {
 			HAWQJdbcUtils.closeConnection(conn);

http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/28d192d2/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/ut/HAWQInputFormatUnitTest_TPCH.java
----------------------------------------------------------------------
diff --git a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/ut/HAWQInputFormatUnitTest_TPCH.java
b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/ut/HAWQInputFormatUnitTest_TPCH.java
index 0a7d32c..9df1940 100644
--- a/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/ut/HAWQInputFormatUnitTest_TPCH.java
+++ b/contrib/hawq-hadoop/hawq-mapreduce-tool/src/test/java/com/pivotal/hawq/mapreduce/ut/HAWQInputFormatUnitTest_TPCH.java
@@ -23,30 +23,35 @@ package com.pivotal.hawq.mapreduce.ut;
 import com.pivotal.hawq.mapreduce.TPCHLocalTester;
 import com.pivotal.hawq.mapreduce.metadata.HAWQTableFormat;
 import org.junit.Test;
+import org.junit.Ignore;
 
 /**
  * Test reading TPC-H table using HAWQInputFormat
  */
 public class HAWQInputFormatUnitTest_TPCH extends TPCHLocalTester {
 
+	@Ignore("dbgen missing")
 	@Test
 	public void testTPCH_AO_No_Partition() throws Exception {
 		HAWQTPCHSpec tpchSpec = new HAWQTPCHSpec("0.001", HAWQTableFormat.AO, false);
 		testTPCHTable(tpchSpec, "lineitem_ao_row");
 	}
 
+	@Ignore("dbgen missing")
 	@Test
 	public void testTPCH_AO_Partition() throws Exception {
 		HAWQTPCHSpec tpchSpec = new HAWQTPCHSpec("0.001", HAWQTableFormat.AO, true);
 		testTPCHTable(tpchSpec, "lineitem_ao_row");
 	}
 
+	@Ignore("dbgen missing")
 	@Test
 	public void testTPCH_Parquet_No_Partition() throws Exception {
 		HAWQTPCHSpec tpchSpec = new HAWQTPCHSpec("0.001", HAWQTableFormat.Parquet, false);
 		testTPCHTable(tpchSpec, "lineitem_ao_parquet");
 	}
 
+	@Ignore("dbgen missing")
 	@Test
 	public void testTPCH_Parquet_Partition() throws Exception {
 		HAWQTPCHSpec tpchSpec = new HAWQTPCHSpec("0.001", HAWQTableFormat.Parquet, true);


Mime
View raw message